feat: 导入 IRSTD 研究代码与文档（小目标检测）

- 新增 irstd 核心包、scripts、tests、configs
- 新增研究文档：literature_review / mathematical_analysis / novelty_matrix / experiment_plan / research_decision / research_log / manuscript_outline
- 新增 evidence、results 数据与审计产物
- 新增 pyproject.toml、requirements、.gitignore

scripts/audit_data.py

@@ -0,0 +1,24 @@
+import argparse,json
+from collections import Counter
+from pathlib import Path
+import numpy as np
+from irstd.data import read_manifest,read_pair,sha
+from irstd.core import components,centers
+
+def main():
+ p=argparse.ArgumentParser();p.add_argument('--manifest',required=True);p.add_argument('--out',required=True);a=p.parse_args()
+ rows=read_manifest(a.manifest);areas=[];dist=[];sizes=Counter();counts=[];empty=0
+ for r in rows:
+  x,y=read_pair(r);ms=components(y);areas.extend(int(m.sum()) for m in ms);counts.append(len(ms));empty+=not len(ms)
+  sizes[str(x.shape)]+=1;c=centers(ms)
+  if len(c)>1:
+   d=np.linalg.norm(c[:,None]-c[None,:],axis=-1);np.fill_diagonal(d,np.inf);dist.extend(d.min(1).tolist())
+ def quant(v):return dict(zip(['min','q25','median','q75','max'],map(float,np.quantile(v,[0,.25,.5,.75,1])))) if v else None
+ result=dict(manifest_sha256=sha(a.manifest),splits=dict(Counter(r['split'] for r in rows)),
+             groups=len(set(r['group'] for r in rows)),resolutions=dict(sizes),empty_images=empty,
+             target_area_pixels=quant(areas),targets_per_image=quant(counts),nearest_target_distance=quant(dist),
+             area_bins={'1-4':sum(1<=v<=4 for v in areas),'5-9':sum(5<=v<=9 for v in areas),'10-25':sum(10<=v<=25 for v in areas),'26+':sum(v>=26 for v in areas)},
+             image_hashes={r['id']:sha(r['image']) for r in rows},mask_hashes={r['id']:sha(r['mask']) for r in rows},
+             limitations='Does not verify licensing, missing annotations, visual near-duplicates, source identities, or independence.')
+ Path(a.out).write_text(json.dumps(result,indent=2))
+if __name__=='__main__':main()

scripts/benchmark.py

@@ -0,0 +1,37 @@
+"""Hardware measurement only; random input is NOT accuracy evaluation."""
+import argparse,json,time,statistics
+from pathlib import Path
+import torch
+from irstd.models import TinyUNet
+from irstd.core import candidate_pool
+
+def main():
+ p=argparse.ArgumentParser();p.add_argument('--out',required=True);p.add_argument('--size',type=int,default=256)
+ p.add_argument('--device',default='cuda');p.add_argument('--iterations',type=int,default=100);a=p.parse_args()
+ if a.iterations<1 or a.size<2:p.error('Invalid benchmark dimensions')
+ model=TinyUNet().to(a.device).eval();x=torch.randn(1,1,a.size,a.size,device=a.device)
+ cuda=str(a.device).startswith('cuda')
+ def sync():
+  if cuda:torch.cuda.synchronize()
+ if cuda:torch.cuda.reset_peak_memory_stats()
+ times=[];full=[]
+ with torch.no_grad():
+  for i in range(20+a.iterations):
+   sync();start=time.perf_counter();y=model(x).sigmoid();sync();middle=time.perf_counter()
+   candidate_pool(y[0,0].cpu().numpy());end=time.perf_counter()
+   if i>=20:times.append((middle-start)*1000);full.append((end-start)*1000)
+ # Exact conv MAC estimate via hooks, measured for declared resolution.
+ macs=[0]
+ def hook(m,inp,out):
+  macs[0]+=out.numel()*(m.in_channels//m.groups)*m.kernel_size[0]*m.kernel_size[1]
+ handles=[m.register_forward_hook(hook) for m in model.modules() if isinstance(m,torch.nn.Conv2d)]
+ with torch.no_grad():model(x)
+ for h in handles:h.remove()
+ Path(a.out).write_text(json.dumps(dict(device=a.device,gpu=torch.cuda.get_device_name() if cuda else None,
+  input=[1,1,a.size,a.size],dtype='float32',warmup=20,iterations=a.iterations,
+  parameters=sum(p.numel() for p in model.parameters()),conv_MACs=macs[0],conv_FLOPs_2_per_MAC=2*macs[0],
+  model_latency_median_ms=statistics.median(times),end_to_end_median_ms=statistics.median(full),
+  peak_allocated_bytes=torch.cuda.max_memory_allocated() if cuda else None,
+  excludes='Disk IO, candidate MLP, graph solver; FLOPs exclude interpolation/activations/pooling',
+  accuracy='not measured; random input'),indent=2))
+if __name__=='__main__':main()

scripts/check_math.py

@@ -0,0 +1,16 @@
+import json,math,platform
+from pathlib import Path
+import numpy as np
+import scipy
+from irstd.core import cp_upper,calibrate
+m=101;delta=.05;alpha=.01;k=64
+n_any=math.ceil(math.log(delta/m)/math.log1p(-alpha))
+n_count=math.ceil(math.log(delta/(m*k))/math.log1p(-alpha/k))
+report=dict(experiment_type='analytic_sample_size_and_synthetic_sanity_only',python=platform.python_version(),numpy=np.__version__,scipy=scipy.__version__,
+ n_zero_error_any=n_any,n_zero_error_count_tail_sum=n_count,
+ n_zero_error_any_fixed_sequence=math.ceil(math.log(delta)/math.log1p(-alpha)),
+ n_zero_error_count_fixed_sequence=math.ceil(math.log(delta)/math.log1p(-alpha/k)),
+ any_upper_n80=cp_upper(0,80,delta/m),
+ any_upper_n1000=cp_upper(0,1000,delta/m),
+ detector_training='not run',real_dataset_metrics='not measured')
+Path('results/math_checks.json').write_text(json.dumps(report,indent=2));print(json.dumps(report,indent=2))

scripts/evaluate.py

@@ -0,0 +1,78 @@
+"""Calibrate/evaluate frozen candidates. No training or test threshold selection."""
+import argparse,json,hashlib
+from pathlib import Path
+import numpy as np
+from irstd.core import candidate_pool,image_stats,aggregate,calibrate,components
+from irstd.data import read_manifest,read_pair,sha
+
+def main():
+ p=argparse.ArgumentParser();p.add_argument('--manifest',required=True);p.add_argument('--predictions',required=True)
+ p.add_argument('--out',required=True);p.add_argument('--gate');p.add_argument('--split',choices=['dev','cal','test'],default='test')
+ p.add_argument('--calibrate',action='store_true');p.add_argument('--certificate');p.add_argument('--threshold',type=float,default=.5)
+ p.add_argument('--base-threshold',type=float,default=.1);p.add_argument('--cap',type=int,default=64);p.add_argument('--radius',type=float,default=3.)
+ p.add_argument('--alpha',type=float,default=.1);p.add_argument('--delta',type=float,default=.05)
+ p.add_argument('--risk',choices=['count','any'],default='count');p.add_argument('--pixel-baseline',action='store_true')
+ p.add_argument('--calibration-method',choices=['bonferroni','fixed_sequence'],default='bonferroni')
+ a=p.parse_args()
+ if a.calibrate and (a.split!='cal' or a.pixel_baseline or a.certificate):p.error('Calibration requires cal split, fixed candidates and no certificate input')
+ if a.pixel_baseline and (a.gate or a.certificate):p.error('Pixel baseline cannot use gate/candidate certificate')
+ if not 0<=a.threshold<=1:p.error('threshold outside [0,1]')
+ rows=read_manifest(a.manifest);root=Path(a.predictions)
+ provenance_path=root/'provenance.json'
+ if not provenance_path.exists():raise ValueError('Missing prediction provenance.json')
+ provenance=json.loads(provenance_path.read_text())
+ if provenance.get('manifest_sha256')!=sha(a.manifest):raise ValueError('Prediction manifest differs')
+ pipeline=dict(manifest_sha256=sha(a.manifest),provenance=provenance,
+               base_threshold=a.base_threshold,cap=a.cap,radius=a.radius,
+               gate_sha256=sha(a.gate) if a.gate else None,
+               prediction_hashes={r['id']:sha(root/(r['id']+'.npy')) for r in rows},
+               input_hashes={r['id']:dict(image=sha(r['image']),mask=sha(r['mask'])) for r in rows},
+               source_hashes={name:sha(Path(__file__).resolve().parents[1]/name) for name in ['irstd/core.py','irstd/models.py','scripts/evaluate.py']},
+               evaluation_version=1)
+ fingerprint=hashlib.sha256(json.dumps(pipeline,sort_keys=True).encode()).hexdigest()
+ gate=None
+ if a.gate:
+  import torch
+  from irstd.models import CandidateGate
+  ck=torch.load(a.gate,map_location='cpu',weights_only=True);cfg=ck['config']
+  for k in ('cap','base_threshold','radius'):
+   if cfg[k]!=getattr(a,k):raise ValueError('Gate pipeline differs: '+k)
+  if cfg['manifest_sha256']!=sha(a.manifest):raise ValueError('Gate manifest differs')
+  gate=CandidateGate();gate.load_state_dict(ck['state_dict']);gate.eval()
+ thresholds=np.linspace(0,1,101).tolist() if a.calibrate else [a.threshold]
+ reject_all=False
+ if a.certificate:
+  cert=json.loads(Path(a.certificate).read_text())
+  if cert['pipeline_fingerprint']!=fingerprint:raise ValueError('Certificate invalidated by changed pipeline/data')
+  reject_all=cert['reject_all'];thresholds=[cert['threshold'] if not reject_all else 1.]
+ selected=[r for r in rows if r['split']==a.split]
+ if not selected:raise ValueError('Empty split')
+ fp=[];stats=[];per_image=[]
+ for r in selected:
+  prob=np.load(root/(r['id']+'.npy'),allow_pickle=False);_,gt=read_pair(r)
+  if prob.shape!=gt.shape:raise ValueError('Prediction shape mismatch')
+  masks,features=candidate_pool(prob,a.base_threshold,a.cap)
+  scores=features[:,0]
+  if gate is not None and len(features):
+   with torch.no_grad():scores=gate(torch.from_numpy(features)).sigmoid().numpy()
+  row_stats=[]
+  for t in thresholds:
+   keep=[] if reject_all else [m for m,s in zip(masks,scores) if s>=t]
+   if a.pixel_baseline:keep=components(prob>=t)
+   row_stats.append(image_stats(keep,gt,a.radius))
+  fp.append([s['fp'] for s in row_stats]);stats.append(row_stats[0])
+  per_image.append(dict(id=r['id'],group=r['group'],**row_stats[0]))
+ if a.calibrate:
+  result=calibrate(fp,thresholds,a.cap,a.alpha,a.delta,a.risk,a.calibration_method)
+  result.update(pipeline_fingerprint=fingerprint,pipeline=pipeline,
+                calibration_ids=[r['id'] for r in selected],
+                calibration_groups=len(set(r['group'] for r in selected)),
+                assumption_notice='Conditional on iid images; group labels alone do not verify independence. Do not pool adjacent video frames.')
+ else:
+  result=dict(metrics=aggregate(stats),per_image=per_image,pipeline_fingerprint=fingerprint,
+              split=a.split,threshold=None if reject_all else thresholds[0],reject_all=reject_all,
+              certificate_used=a.certificate,mode='pixel' if a.pixel_baseline else 'fixed_candidate')
+ dest=Path(a.out);dest.parent.mkdir(parents=True,exist_ok=True)
+ if dest.exists():raise FileExistsError(dest)
+ dest.write_text(json.dumps(result,indent=2,allow_nan=False));print(dest)
+if __name__=='__main__':main()

scripts/predict.py

@@ -0,0 +1,20 @@
+import argparse,json
+from pathlib import Path
+import numpy as np
+import torch
+from irstd.models import TinyUNet
+from irstd.data import read_manifest,read_pair,sha
+
+def main():
+ p=argparse.ArgumentParser();p.add_argument('--manifest',required=True);p.add_argument('--checkpoint',required=True)
+ p.add_argument('--out',required=True);p.add_argument('--device',default='cuda');a=p.parse_args()
+ ck=torch.load(a.checkpoint,map_location='cpu',weights_only=True)
+ model=TinyUNet().to(a.device);model.load_state_dict(ck['state_dict']);model.eval()
+ out=Path(a.out);out.mkdir(parents=True,exist_ok=False)
+ with torch.no_grad():
+  for r in read_manifest(a.manifest):
+   x,_=read_pair(r)
+   prob=model(torch.from_numpy(x)[None,None].to(a.device)).sigmoid()[0,0].cpu().numpy()
+   np.save(out/(r['id']+'.npy'),prob)
+ (out/'provenance.json').write_text(json.dumps(dict(checkpoint_sha256=sha(a.checkpoint),manifest_sha256=sha(a.manifest)),indent=2))
+if __name__=='__main__':main()

scripts/train.py

@@ -0,0 +1,40 @@
+"""Single-device, batch-one reference training; no dataset auto-download."""
+import argparse,json,random,os
+from pathlib import Path
+os.environ.setdefault('CUBLAS_WORKSPACE_CONFIG', ':4096:8')
+import numpy as np
+import torch
+from irstd.models import TinyUNet,segmentation_loss
+from irstd.data import read_manifest,read_pair,sha
+
+def main():
+ p=argparse.ArgumentParser(); p.add_argument('--manifest',required=True);p.add_argument('--out',required=True)
+ p.add_argument('--epochs',type=int,default=100);p.add_argument('--seed',type=int,default=0)
+ p.add_argument('--loss',choices=['bce','dice','bce_dice','focal','iou'],default='bce_dice')
+ p.add_argument('--device',default='cuda'); a=p.parse_args()
+ if a.epochs<1: p.error('epochs must be positive')
+ random.seed(a.seed);np.random.seed(a.seed);torch.manual_seed(a.seed)
+ torch.use_deterministic_algorithms(True);torch.backends.cudnn.benchmark=False
+ rows=read_manifest(a.manifest);tr=[r for r in rows if r['split']=='train'];dv=[r for r in rows if r['split']=='dev']
+ if not tr or not dv: raise ValueError('Train and development splits required')
+ out=Path(a.out);out.mkdir(parents=True,exist_ok=False)
+ model=TinyUNet().to(a.device);opt=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-4)
+ config=vars(a)|dict(manifest_sha256=sha(a.manifest),torch=str(torch.__version__),numpy=np.__version__,cuda=torch.version.cuda)
+ (out/'config.json').write_text(json.dumps(config,indent=2));best=float('inf')
+ def batch(r):
+  x,y=read_pair(r)
+  return torch.from_numpy(x)[None,None].to(a.device),torch.from_numpy(y.astype(np.float32))[None,None].to(a.device)
+ with (out/'train.jsonl').open('w') as f:
+  for epoch in range(a.epochs):
+   model.train();random.shuffle(tr);total=0.
+   for r in tr:
+    x,y=batch(r);opt.zero_grad();loss=segmentation_loss(model(x),y,a.loss)
+    loss.backward();opt.step();total+=loss.item()
+   model.eval();dev=0.
+   with torch.no_grad():
+    for r in dv:
+     x,y=batch(r);dev+=segmentation_loss(model(x),y,a.loss).item()
+   dev/=len(dv);f.write(json.dumps(dict(epoch=epoch,train_loss=total/len(tr),dev_loss=dev))+'\n');f.flush()
+   if dev<best:
+    best=dev;torch.save(dict(state_dict=model.state_dict(),config=config,epoch=epoch),out/'best.pt')
+if __name__=='__main__':main()

scripts/train_gate.py

@@ -0,0 +1,48 @@
+import argparse,json,random,os
+from pathlib import Path
+os.environ.setdefault('CUBLAS_WORKSPACE_CONFIG', ':4096:8')
+import numpy as np
+import torch
+from torch.nn import functional as F
+from irstd.models import CandidateGate,gate_loss
+from irstd.core import candidate_pool,canonical_labels
+from irstd.data import read_manifest,read_pair,sha
+
+def main():
+ p=argparse.ArgumentParser();p.add_argument('--manifest',required=True);p.add_argument('--predictions',required=True);p.add_argument('--out',required=True)
+ p.add_argument('--epochs',type=int,default=100);p.add_argument('--seed',type=int,default=0)
+ p.add_argument('--mode',choices=['bce','constrained'],default='constrained');p.add_argument('--no-size',action='store_true')
+ p.add_argument('--alpha-train',type=float,default=.1);p.add_argument('--cap',type=int,default=64);p.add_argument('--base-threshold',type=float,default=.1)
+ p.add_argument('--radius',type=float,default=3.);a=p.parse_args()
+ if a.epochs<1 or a.alpha_train<0: p.error('Invalid epochs/budget')
+ random.seed(a.seed);np.random.seed(a.seed);torch.manual_seed(a.seed);torch.use_deterministic_algorithms(True)
+ rows=read_manifest(a.manifest);cached=[];files={}
+ for r in rows:
+  if r['split']!='train':continue
+  path=Path(a.predictions)/(r['id']+'.npy');prob=np.load(path,allow_pickle=False)
+  _,gt=read_pair(r)
+  if prob.shape!=gt.shape:raise ValueError('Prediction shape mismatch')
+  masks,features=candidate_pool(prob,a.base_threshold,a.cap);labels,areas=canonical_labels(masks,gt,a.radius)
+  cached.append(tuple(torch.from_numpy(v) for v in (features,labels,areas)));files[r['id']]=sha(path)
+ if not cached:raise ValueError('No train images')
+ gate=CandidateGate();opt=torch.optim.Adam(gate.parameters(),lr=1e-3);dual=0.
+ out=Path(a.out);out.mkdir(parents=True,exist_ok=False)
+ config=vars(a)|dict(manifest_sha256=sha(a.manifest),training_prediction_hashes=files)
+ # Fixed epochs: tune this on dev in separate runs, never on calibration/test labels.
+ with (out/'gate.jsonl').open('w') as f:
+  for epoch in range(a.epochs):
+   random.shuffle(cached);upper_total=0.;loss_total=0.
+   for x,y,area in cached:
+    if not len(x):continue  # zero surrogate contribution; retain image in denominator
+    opt.zero_grad();s=gate(x)
+    if a.mode=='constrained':
+     loss,upper=gate_loss(s,y,area,dual,a.alpha_train,not a.no_size)
+    else:
+     loss=F.binary_cross_entropy_with_logits(s,y);_,upper=gate_loss(s,y,area,0.,a.alpha_train)
+    loss.backward();opt.step();upper_total+=float(upper.detach());loss_total+=float(loss.detach())
+   mean_upper=upper_total/len(cached)
+   if a.mode=='constrained':dual=max(0.,dual+.01*(mean_upper-a.alpha_train))
+   f.write(json.dumps(dict(epoch=epoch,loss=loss_total/len(cached),surrogate=mean_upper,dual=dual))+'\n');f.flush()
+ torch.save(dict(state_dict=gate.state_dict(),config=config),out/'gate.pt')
+ (out/'config.json').write_text(json.dumps(config,indent=2))
+if __name__=='__main__':main()