scripts/build_progressive.py

"""Build packed, geometry-first derivatives of the already-public homepage models."""
import concurrent.futures, copy, hashlib, io, json, struct, urllib.request
from pathlib import Path
from PIL import Image
ROOT=Path(__file__).resolve().parent.parent
OUT=ROOT/'models/progressive-v1';OUT.mkdir(parents=True,exist_ok=True)
import tempfile
CACHE=Path(tempfile.gettempdir())/'c4s-progressive-models';CACHE.mkdir(exist_ok=True)
KEY=bytes.fromhex('5c2f9e71d3a84b06e1f7c29a38d05b64a7193ce8f2046db95e81c73a2df6904b')
def pack(b):return bytes(v^KEY[i&31]^(((i>>5)*131)&255) for i,v in enumerate(b))
def convert(url):
 name=url.split('/')[-1].split('?')[0];p=CACHE/name
 if not p.exists():
  with urllib.request.urlopen('https://simworld-ai-code4scene.static.hf.space/'+url,timeout=90) as r:p.write_bytes(r.read())
 original=pack(p.read_bytes());assert original[:4]==b'glTF'
 n=struct.unpack_from('<I',original,12)[0];g=json.loads(original[20:20+n]);before=copy.deepcopy(g)
 binary=original[28+n:];assert len(binary)==struct.unpack_from('<I',original,20+n)[0]
 image_views={im['bufferView'] for im in g.get('images',[])};payload=bytearray(b'\0'*4);ranges={}
 def move(offset,length):
  k=(offset,length)
  if k not in ranges:
   while len(payload)%4:payload.append(0)
   ranges[k]=len(payload);payload.extend(binary[offset:offset+length])
  return ranges[k]
 for i,v in enumerate(g['bufferViews']):
  if i in image_views:v.update(buffer=0,byteOffset=0,byteLength=4);continue
  if v['buffer']==0:v['byteOffset']=move(v.get('byteOffset',0),v['byteLength'])
  ext=v.get('extensions',{}).get('EXT_meshopt_compression')
  if ext and ext['buffer']==0:ext['byteOffset']=move(ext.get('byteOffset',0),ext['byteLength'])
 images=[]
 for i,im in enumerate(g.get('images',[])):
  v=before['bufferViews'][im['bufferView']];b=binary[v.get('byteOffset',0):v.get('byteOffset',0)+v['byteLength']]
  with Image.open(io.BytesIO(b)) as img:
   img.load()
   if max(img.size)>512:
    img.thumbnail((512,512),Image.Resampling.LANCZOS);dest=io.BytesIO();img.save(dest,format='WEBP',quality=85,method=4);b=dest.getvalue()
   width,height=img.size
  images.append((i,b,width,height));del im['bufferView'];im['uri']='deferred-image-'+str(i);im['mimeType']='image/webp'
 texture_bytes=0;bundles=[]
 # Small batches avoid hundreds of HTTP requests and permit visible incremental updates.
 for start in range(0,len(images),32):
  group=images[start:start+32];data=b''.join(x[1] for x in group);digest=hashlib.sha256(data).hexdigest()[:24];filename='tex-'+digest+'.bin'
  (OUT/filename).write_bytes(pack(data));bundles.append(filename);texture_bytes+=len(data);offset=0
  for i,b,w,h in group:
   g['images'][i].setdefault('extras',{})['c4sTexture']={'bundle':filename,'offset':offset,'length':len(b),'width':w,'height':h};offset+=len(b)
 while len(payload)%4:payload.append(0)
 g['buffers'][0]['byteLength']=len(payload)
 header=json.dumps(g,separators=(',',':'),ensure_ascii=False).encode();header+=b' '*((-len(header))%4)
 glb=struct.pack('<III',0x46546c67,2,28+len(header)+len(payload))+struct.pack('<II',len(header),0x4e4f534a)+header+struct.pack('<II',len(payload),0x004e4942)+payload
 filename='geo-'+hashlib.sha256(glb).hexdigest()[:24]+'.bin';(OUT/filename).write_bytes(pack(glb))
 # Verify all geometry bytes (including compressed meshopt ranges) are identical.
 for i,v in enumerate(before['bufferViews']):
  if i in image_views:continue
  for a,b in [(v,g['bufferViews'][i]),(v.get('extensions',{}).get('EXT_meshopt_compression'),g['bufferViews'][i].get('extensions',{}).get('EXT_meshopt_compression'))]:
   if a and a['buffer']==0:assert binary[a.get('byteOffset',0):a.get('byteOffset',0)+a['byteLength']]==payload[b.get('byteOffset',0):b.get('byteOffset',0)+b['byteLength']]
 result={'geometry':'models/progressive-v1/'+filename,'originalBytes':len(original),'geometryBytes':len(glb),'textureBytes':texture_bytes,'images':len(images),'bundles':len(bundles)}
 print(name,result,flush=True);return name,result
if __name__=='__main__':
 urls=json.loads((ROOT/'scripts/progressive-card-urls.json').read_text())
 with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:manifest=dict(pool.map(convert,urls))
 (ROOT/'data/progressive-scenes.json').write_text(json.dumps(manifest,indent=2)+'\n')
 print('TOTAL',sum(x['originalBytes'] for x in manifest.values()),sum(x['geometryBytes'] for x in manifest.values()),sum(x['textureBytes'] for x in manifest.values()),flush=True)