import importlib.util import json from pathlib import Path import tempfile import time import unittest spec=importlib.util.spec_from_file_location('dashboard',Path(__file__).resolve().parents[1]/'server.py') m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) class DashboardTests(unittest.TestCase): def setUp(self): self.temp=tempfile.TemporaryDirectory(prefix='dashboard-qa-') self.old=(m.DATA,m.LOGS) m.DATA=Path(self.temp.name)/'state';m.LOGS=Path(self.temp.name)/'logs';m.LOGS.mkdir() m.initialize() m.LATEST.clear();m.LATEST.update({'ts':time.time(),'boot_id':'test-current','monotonic_s':1000,'issues':[]}) def tearDown(self): m.DATA,m.LOGS=self.old;self.temp.cleanup() def test_import_replay_and_clock_adjustment(self): now=int(time.time()) row={'observed_at':m.iso(now),'first_at':m.iso(now+3600),'last_at':m.iso(now-3600), 'first_monotonic_us':'11000000','last_monotonic_us':'487000000', 'boot_id':'oldboot','source':'kernel','category':'power','severity':'ERROR','count':93, 'device':'0000:84:00.0','detail':'GPU does not have the necessary power cables connected.'} p=m.LOGS/'events-test.jsonl';p.write_text(json.dumps(row)+'\n'+json.dumps(row)+'\n') with m.connect() as db: m.import_logs(db);m.import_logs(db) self.assertEqual(db.execute('SELECT COUNT(*) FROM events').fetchone()[0],1) result=m.events({'days':['7']}) self.assertEqual(result['total'],1) self.assertEqual(result['items'][0]['state'],'history') self.assertAlmostEqual(result['items'][0]['first_ts'],now-3600-476,delta=1) self.assertTrue(result['items'][0]['time_note']) self.assertEqual(result['items'][0]['raw']['first_at'],row['first_at']) def test_history_does_not_invent_old_cpu_utilization(self): now=int(time.time())-40 with m.connect() as db: db.execute('INSERT INTO metrics VALUES(?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,?,?)', (now-1500,json.dumps([{'uuid':'gpu-a','name':'P100','temperature_c':40,'power_w':30}]),'legacy')) for offset,cpu in [(0,40),(10,80)]: db.execute('INSERT INTO metrics VALUES(?,?,?,?,?,?,?,?,?,?)', (now+offset,cpu,60,45,1,2,3,4,json.dumps([{'uuid':'gpu-a','name':'P100','utilization':90,'temperature_c':50,'power_w':100}]),'live')) data=m.history('1h') legacy=[p for p in data['points'] if not p['live']] self.assertTrue(legacy) self.assertIsNone(legacy[0]['cpu']) self.assertIsNone(legacy[0]['gpus']['gpu-a']['utilization']) self.assertEqual(legacy[0]['gpus']['gpu-a']['temperature_c'],40) self.assertEqual(data['devices']['gpu-a'],'P100') self.assertTrue(any(p['cpu'] is not None for p in data['points'])) def test_filters_and_partial_tail(self): now=int(time.time()) row={'observed_at':m.iso(now),'category':'storage','severity':'ERROR','detail':'nvme timeout'} p=m.LOGS/'events-tail.jsonl';p.write_text(json.dumps(row)) with m.connect() as db: m.import_logs(db) self.assertEqual(db.execute('SELECT COUNT(*) FROM events').fetchone()[0],0) with p.open('a') as f:f.write('\n') with m.connect() as db:m.import_logs(db) self.assertEqual(m.events({'category':['storage']})['total'],1) self.assertEqual(m.events({'category':['power']})['total'],0) self.assertEqual(m.events({'q':['timeout']})['total'],1) def test_stale_status_is_unknown_not_recovered(self): m.LATEST.update({'ts':time.time()-300}) now=int(time.time()) row={'observed_at':m.iso(now),'category':'power','severity':'ERROR','detail':'power error'} (m.LOGS/'events-stale.jsonl').write_text(json.dumps(row)+'\n') with m.connect() as db:m.import_logs(db) self.assertEqual(m.events({})['items'][0]['state'],'unknown') def test_link_reports_merge_and_keep_two_cards_separate(self): now=int(time.time()) a={'id':'gpu-a','label':'第 1 张 P100','pci':'0000:03:00.0'} b={'id':'gpu-b','label':'第 2 张 P100','pci':'0000:84:00.0'} m.LATEST.update({'device_map':{'0000:00:03.0':a,'0000:03:00.0':a,'0000:80:03.0':b}, 'gpus':[{'uuid':'gpu-a'},{'uuid':'gpu-b'}],'expected_gpu_count':2}) def row(device,detail,timestamp,mono): return {'observed_at':m.iso(now),'first_at':m.iso(timestamp),'last_at':m.iso(timestamp), 'first_monotonic_us':str(mono),'last_monotonic_us':str(mono), 'boot_id':'test-current','source':'kernel','category':'pcie_correctable','severity':'WARN', 'device':device,'count':1,'detail':detail} rows=[row('0000:00:03.0','PCIe Bus Error: severity=Correctable',now-50,950000000), row('0000:00:03.0','[6] BadTLP',now-50,950000001), row('0000:03:00.0','PCIe Bus Error: severity=Correctable',now-50,950000002), row('0000:80:03.0','PCIe Bus Error: severity=Correctable',now-40,960000000)] path=m.LOGS/'events-grouped.jsonl';path.write_text(''.join(json.dumps(x)+'\n' for x in rows)) with m.connect() as db:m.import_logs(db) result=m.events({}) self.assertEqual(result['raw_total'],4) self.assertEqual(result['total'],2) first=next(x for x in result['items'] if x['component_id']=='gpu-a') self.assertEqual(first['record_count'],3) self.assertEqual(first['signal_reports'],2) self.assertEqual(m.events({'view':['raw']})['total'],4) original_id=first['id'] with path.open('a') as f:f.write(json.dumps(row('0000:03:00.0','PCIe Bus Error: severity=Correctable',now-10,990000000))+'\n') with m.connect() as db:m.import_logs(db) new=next(x for x in m.events({})['items'] if x['component_id']=='gpu-a') self.assertEqual(new['id'],original_id) self.assertEqual(new['signal_reports'],3) def test_five_minute_quiet_gap_starts_another_incident(self): now=int(time.time()) rows=[{'observed_at':m.iso(now),'first_at':m.iso(now-age),'last_at':m.iso(now-age), 'source':'kernel','boot_id':'oldboot','first_monotonic_us':str(1000000+i), 'last_monotonic_us':str(1000000+i),'device':'0000:03:00.0', 'category':'pcie_correctable','severity':'WARN','count':1, 'detail':'PCIe Bus Error: severity=Correctable'} for i,age in enumerate([1000,10])] (m.LOGS/'events-gaps.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in rows)) with m.connect() as db:m.import_logs(db) self.assertEqual(m.events({})['total'],2) def test_current_retry_is_warning_and_has_plain_action(self): now=int(time.time()) m.LATEST.update({'gpus':[{'uuid':'gpu-a'},{'uuid':'gpu-b'}],'expected_gpu_count':2, 'issues':[{'category':'pcie_correctable','device':'0000:00:03.0'}], 'device_map':{'0000:00:03.0':{'id':'gpu-a','label':'第 1 张 P100','pci':'0000:03:00.0'}}}) row={'observed_at':m.iso(now),'first_at':m.iso(now-10),'last_at':m.iso(now-10), 'source':'kernel','boot_id':'test-current','first_monotonic_us':'990000000', 'last_monotonic_us':'990000000','device':'0000:00:03.0', 'category':'pcie_correctable','severity':'WARN','count':1, 'detail':'PCIe Bus Error: severity=Correctable'} (m.LOGS/'events-active.jsonl').write_text(json.dumps(row)+'\n') with m.connect() as db:m.import_logs(db) item=m.events({})['items'][0] self.assertEqual(item['severity'],'WARN') self.assertEqual(item['state'],'active') self.assertIn('显卡可用',item['state_label']) self.assertIn('不能证明',item['meaning']) self.assertEqual(len(item['next_steps']),3) def test_problem_filter_hides_routine_information(self): now=int(time.time()) rows=[{'observed_at':m.iso(now),'category':cat,'severity':severity,'detail':cat} for cat,severity in [('boot','INFO'),('power','ERROR')]] (m.LOGS/'events-levels.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in rows)) with m.connect() as db:m.import_logs(db) self.assertEqual(m.events({'severity':['problem']})['total'],1) self.assertEqual(m.events({'severity':['INFO']})['total'],1) if __name__=='__main__':unittest.main(verbosity=2)