Spaces:
Configuration error
Configuration error
| import os | |
| import json | |
| from . import utils | |
| default_models_info = { | |
| "checkpoints/Alternative/hunyuan_dit_1.2.safetensors": { | |
| "size": 8240228270, | |
| "hash": "4fb84f84079cda457d171b3c6b15d1be95b5a3e5d9825703951a99ddf92d1787", | |
| "muid": "8c4c0098ac" | |
| }, | |
| "checkpoints/Alternative/playground-v2.5-1024px.safetensors": { | |
| "size": 6938040576, | |
| "hash": "bcaa7dd6780974f000b17b5a6c63e6f867a75c51ffa85c67d6b196882c69b992", | |
| "muid": "d0e21c789d50" | |
| }, | |
| "checkpoints/albedobaseXL_v21.safetensors": { | |
| "size": 6938041042, | |
| "hash": "1718b5bb2da1ef4815fee8af8a7fc2fa8ab8f467b279eded4d991ea0cce59a6d", | |
| "muid": "acf28f1aeb42" | |
| }, | |
| "checkpoints/animaPencilXL_v310.safetensors": { | |
| "size": 6938040682, | |
| "hash": "67b97ee6eec64abf0cb73c2284a0afecdd8e205a87a3622c2c23231e25e29b5b", | |
| "muid": "2def60f4b273" | |
| }, | |
| "checkpoints/animaPencilXL_v500.safetensors": { | |
| "size": 6938041144, | |
| "hash": "896faa18cd6852ccf977e2dec76191c38f256d031204e233cb3ed76f6088d55b", | |
| "muid": "239e9199aa" | |
| }, | |
| "checkpoints/FluxDev/flux1-dev.safetensors": { | |
| "size": 23802932552, | |
| "hash": "4610115bb0c89560703c892c59ac2742fa821e60ef5871b33493ba544683abd7", | |
| "muid": "2f3c5caac0" | |
| }, | |
| "checkpoints/FluxDev/flux1-dev-fp8.safetensors": { | |
| "size": 17246524772, | |
| "hash": "8e91b68084b53a7fc44ed2a3756d821e355ac1a7b6fe29be760c1db532f3d88a", | |
| "muid": "7f89b4dd65" | |
| }, | |
| "checkpoints/FluxDev/flux1-dev-bnb-nf4.safetensors": { | |
| "size": 11489884113, | |
| "hash": "c5e25d12d720e30a277598ce9ded9db406ee54f63419fe0c801b283d4ea146e2", | |
| "muid": "6487417fee" | |
| }, | |
| "checkpoints/FluxDev/flux1-dev-bnb-nf4-v2.safetensors": { | |
| "size": 12044280207, | |
| "hash": "fef37763b8f2c8cc3463139bbb6d91aa517c605b654d4e5c540de52813f30306", | |
| "muid": "8c53267ff9" | |
| }, | |
| "checkpoints/FluxDev/flux-hyp8-Q5_K_M.gguf": { | |
| "size": 8421981408, | |
| "hash": "a0dac309ffb497fde0d1bbfa0291f5371d0d05c66173df830318bc475777c68a", | |
| "muid": "a0dac309ff" | |
| }, | |
| "checkpoints/FluxDev/fluxunchained-dev-Q5_K_M.gguf": { | |
| "size": 8428152160, | |
| "hash": "ee5df99febf1aebd63137672d3396407a6e89f771ca9e83bc13c475c5d57a521", | |
| "muid": "ee5df99feb" | |
| }, | |
| "checkpoints/FluxSchnell/flux1-schnell.safetensors": { | |
| "size": 23782506688, | |
| "hash": "9403429e0052277ac2a87ad800adece5481eecefd9ed334e1f348723621d2a0a", | |
| "muid": "d314672fc6" | |
| }, | |
| "checkpoints/FluxSchnell/flux1-schnell-fp8.safetensors": { | |
| "size": 17236328572, | |
| "hash": "ead426278b49030e9da5df862994f25ce94ab2ee4df38b556ddddb3db093bf72", | |
| "muid": "8f031d049d" | |
| }, | |
| "checkpoints/FLUX.1-schnell-dev-merged.safetensors": { | |
| "size": 23802903480, | |
| "hash": "0dc649761fba58625f57f596738e76422df9424c4c8801ca70c53ad6998c905b", | |
| "muid": "6661979a94" | |
| }, | |
| "checkpoints/FluxSchnell/flux1-schnell-bnb-nf4.safetensors": { | |
| "size": 11484555394, | |
| "hash": "e6cba6afca8b2f5599879111e1a5f3dabebe69bcc3ee4a6af46807447adc6d09", | |
| "muid": "0eaea6dc0d" | |
| }, | |
| "checkpoints/Juggernaut-XL_v9_RunDiffusionPhoto_v2.safetensors": { | |
| "size": 7105348188, | |
| "hash": "c9e3e68f89b8e38689e1097d4be4573cf308de4e3fd044c64ca697bdb4aa8bca", | |
| "muid": "393f61fcec" | |
| }, | |
| "checkpoints/Pony/ponyDiffusionV6XL.safetensors": { | |
| "size": 6938041050, | |
| "hash": "67ab2fd8ec439a89b3fedb15cc65f54336af163c7eb5e4f2acc98f090a29b0b3", | |
| "muid": "e023c14343" | |
| }, | |
| "checkpoints/realisticStockPhoto_v20.safetensors": { | |
| "size": 6938054242, | |
| "hash": "f99f3dec38a09b4834a4a073bdc45aabd42b422b4d327f5e8001afcb5ffb5f45", | |
| "muid": "5d99d6fc4fbf" | |
| }, | |
| "checkpoints/realisticVisionV60B1_v51VAE.safetensors": { | |
| "size": 2132625894, | |
| "hash": "15012c538f503ce2ebfc2c8547b268c75ccdaff7a281db55399940ff1d70e21d", | |
| "muid": "5da06f78b3c8" | |
| }, | |
| "checkpoints/SD3x/sd3_medium_incl_clips.safetensors": { | |
| "size": 5973224240, | |
| "hash": "3bb7f21bc5fb450220f4eb78a2f276b15422309d5166a4bdeb8c3b763a3a0581", | |
| "muid": "bb3cbb3221ef" | |
| }, | |
| "checkpoints/SD3x/sd3_medium_incl_clips_t5xxlfp16.safetensors": { | |
| "size": 15761074532, | |
| "hash": "69a950c5d143ce782a7423c532c8a12b75da6a37b0e6f26a322acf4e76208912", | |
| "muid": "c3a45b17d217" | |
| }, | |
| "checkpoints/SD3x/sd3_medium_incl_clips_t5xxlfp8.safetensors": { | |
| "size": 10867168284, | |
| "hash": "92db4295e9c9ab8401ef60566d975656a35b0bd0f6d9ce0d083725171f7b3174", | |
| "muid": "41d49489bc24" | |
| }, | |
| "checkpoints/sd_xl_base_1.0_0.9vae.safetensors": { | |
| "size": 6938078334, | |
| "hash": "e6bb9ea85bbf7bf6478a7c6d18b71246f22e95d41bcdd80ed40aa212c33cfeff", | |
| "muid": "5e756477ea9d" | |
| }, | |
| "checkpoints/sd_xl_refiner_1.0_0.9vae.safetensors": { | |
| "size": 6075981930, | |
| "hash": "8d0ce6c016004cbdacd50f937dad381d8c396628d621a7f97191470532780164", | |
| "muid": "bd66e233fe56" | |
| }, | |
| "loras/FilmVelvia3.safetensors": { | |
| "size": 151108832, | |
| "hash": "ac8b0e4aa77be4d8b83da9bafe0134a2e36504c9b5263a7030394cffe4f7003a", | |
| "muid": "6e93473d6228" | |
| }, | |
| "loras/Hyper-SDXL-8steps-lora.safetensors": { | |
| "size": 787359648, | |
| "hash": "ca689190e8c46038550384b5675488526cfe5a40d35f82b27acb75c100f417c1", | |
| "muid": "4f494295ed" | |
| }, | |
| "loras/ip-adapter-faceid-plusv2_sdxl_lora.safetensors": { | |
| "size": 371842896, | |
| "hash": "f24b4bb2dad6638a09c00f151cde84991baf374409385bcbab53c1871a30cb7b", | |
| "muid": "13623d29c464" | |
| }, | |
| "loras/sd_xl_offset_example-lora_1.0.safetensors": { | |
| "size": 49553604, | |
| "hash": "4852686128f953d0277d0793e2f0335352f96a919c9c16a09787d77f55cbdf6f", | |
| "muid": "8e3e833226b3" | |
| }, | |
| "loras/SDXL_FILM_PHOTOGRAPHY_STYLE_V1.safetensors": { | |
| "size": 912593164, | |
| "hash": "9e2a98e1f27dbdbb0bda11523dee3444df099599bff7471c6e557f6ad55f27eb", | |
| "muid": "b39d197db0" | |
| }, | |
| "loras/sdxl_hyper_sd_4step_lora.safetensors": { | |
| "size": 787359648, | |
| "hash": "12f81a27d00a751a40d68fd15597091896c5a90f3bd632fb6c475607cbdad76e", | |
| "muid": "1c88f7295856" | |
| }, | |
| "loras/sdxl_lightning_4step_lora.safetensors": { | |
| "size": 393854592, | |
| "hash": "bf56cf2657efb15e465d81402ed481d1e11c4677e4bcce1bc11fe71ad8506b79", | |
| "muid": "1c32bdb07a7c" | |
| }, | |
| "embeddings/unaestheticXLhk1.safetensors": { | |
| "size": 33296, | |
| "hash": "ca29d24a64c1801efc82f8f4d05d98308e5b6c51c15d156fb61ac074f24f87ce", | |
| "muid": "63578af5d493" | |
| }, | |
| "embeddings/unaestheticXLv31.safetensors": { | |
| "size": 33296, | |
| "hash": "75fa9a0423a19c56ccaaea3b985b4999408b530585eca3f6108685c0007e5b2e", | |
| "muid": "a20bca3b2146" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00001-of-00007.bin": { | |
| "size": 1827781090, | |
| "hash": "b6a6388dae55b598efe76c704e7f017bd84e6f6213466b7686a8f8326f78ab05", | |
| "muid": "b6a6388dae" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00002-of-00007.bin": { | |
| "size": 1968299480, | |
| "hash": "2f96bef324acb5c3fe06b7a80f84272fe064d0327cbf14eddfae7af0d665a6ac", | |
| "muid": "2f96bef324" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00003-of-00007.bin": { | |
| "size": 1927415036, | |
| "hash": "2400101255213250d9df716f778b7d2325f2fa4a8acaedee788338fceee5b27e", | |
| "muid": "2400101255" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00004-of-00007.bin": { | |
| "size": 1815225998, | |
| "hash": "472567c1b0e448a19171fbb5b3dab5670426d0a5dfdfd2c3a87a60bb1f96037d", | |
| "muid": "472567c1b0" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00005-of-00007.bin": { | |
| "size": 1968299544, | |
| "hash": "ef2aea78fa386168958e5ba42ecf09cbb567ed3e77ce2be990d556b84081e2b9", | |
| "muid": "ef2aea78fa" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00006-of-00007.bin": { | |
| "size": 1927415036, | |
| "hash": "35191adf21a1ab632c2b175fcbb6c27601150026cb1ed5d602938d825954526f", | |
| "muid": "35191adf21" | |
| }, | |
| "diffusers/Kolors/text_encoder/pytorch_model-00007-of-00007.bin": { | |
| "size": 1052808542, | |
| "hash": "b7cdaa9b8ed183284905c49d19bf42360037fdf2f95acb3093039d3c3a459261", | |
| "muid": "b7cdaa9b8e" | |
| }, | |
| "diffusers/Kolors/unet/diffusion_pytorch_model.fp16.safetensors": { | |
| "size": 5159140240, | |
| "hash": "425ff1dcbe3a70ac13d3afdd69bd4e3176b0c3260722527c80b210f11d2d966c", | |
| "muid": "9c8f088e4c" | |
| }, | |
| "diffusers/Kolors/vae/diffusion_pytorch_model.fp16.safetensors": { | |
| "size": 167335342, | |
| "hash": "bcb60880a46b63dea58e9bc591abe15f8350bde47b405f9c38f4be70c6161e68", | |
| "muid": "345f7343ee" | |
| }, | |
| "DIFFUSERS/Kolors": { | |
| "size": 20054, | |
| "hash": "2eff895bdad33abb2f647c35ccb0c2d70173031effa4f126b335a815442af1e3", | |
| "muid": "2eff895bda" | |
| }, | |
| "controlnet/control-lora-canny-rank128.safetensors": { | |
| "size": 395733680, | |
| "hash": "56389dbb245ca44de91d662529bd4298abc55ce2318f60bc19454fb72ff68247", | |
| "muid": "44f83205f6" | |
| }, | |
| "controlnet/detection_Resnet50_Final.pth": { | |
| "size": 109497761, | |
| "hash": "6d1de9c2944f2ccddca5f5e010ea5ae64a39845a86311af6fdf30841b0a5a16d", | |
| "muid": "6d1de9c294" | |
| }, | |
| "controlnet/fooocus_ip_negative.safetensors": { | |
| "size": 65616, | |
| "hash": "d7caedfb46780825895718c7c8e9ee077e675c935ddfcf272f1c01a4fc8ea72d", | |
| "muid": "4682603510" | |
| }, | |
| "controlnet/fooocus_xl_cpds_128.safetensors": { | |
| "size": 395706528, | |
| "hash": "eec3fd8209a65b41341ea9f415de66909c97b30fb4d20965b3304e8e5251c2f1", | |
| "muid": "aa82117d38" | |
| }, | |
| "controlnet/ip-adapter-plus-face_sdxl_vit-h.bin": { | |
| "size": 1013454761, | |
| "hash": "50e886d82940b3c5873d80c2b06d8a4b0d0fccec70bc44fd53f16ac3cfd7fc36", | |
| "muid": "50e886d829" | |
| }, | |
| "controlnet/ip-adapter-plus_sdxl_vit-h.bin": { | |
| "size": 1013454427, | |
| "hash": "ec70edb7cc8e769c9388d94eeaea3e4526352c9fae793a608782d1d8951fde90", | |
| "muid": "ec70edb7cc" | |
| }, | |
| "controlnet/parsing_bisenet.pth": { | |
| "size": 53289463, | |
| "hash": "468e13ca13a9b43cc0881a9f99083a430e9c0a38abd935431d1c28ee94b26567", | |
| "muid": "468e13ca13" | |
| }, | |
| "controlnet/parsing_parsenet.pth": { | |
| "size": 85331193, | |
| "hash": "3d558d8d0e42c20224f13cf5a29c79eba2d59913419f945545d8cf7b72920de2", | |
| "muid": "3d558d8d0e" | |
| }, | |
| "inpaint/fooocus_inpaint_head.pth": { | |
| "size": 52602, | |
| "hash": "32f7f838e0c6d8f13437ba8411e77a4688d77a2e34df8857e4ef4d51f6b97692", | |
| "muid": "32f7f838e0" | |
| }, | |
| "inpaint/groundingdino_swint_ogc.pth": { | |
| "size": 693997677, | |
| "hash": "3b3ca2563c77c69f651d7bd133e97139c186df06231157a64c507099c52bc799", | |
| "muid": "3b3ca2563c" | |
| }, | |
| "inpaint/sam_vit_b_01ec64.pth": { | |
| "size": 375042383, | |
| "hash": "ec2df62732614e57411cdcf32a23ffdf28910380d03139ee0f4fcbe91eb8c912", | |
| "muid": "ec2df62732" | |
| }, | |
| "inpaint/sam_vit_h_4b8939.pth": { | |
| "size": 2564550879, | |
| "hash": "a7bf3b02f3ebf1267aba913ff637d9a2d5c33d3173bb679e46d9f338c26f262e", | |
| "muid": "a7bf3b02f3" | |
| }, | |
| "inpaint/sam_vit_l_0b3195.pth": { | |
| "size": 1249524607, | |
| "hash": "3adcc4315b642a4d2101128f611684e8734c41232a17c648ed1693702a49a622", | |
| "muid": "3adcc4315b" | |
| }, | |
| "unet/iclight_sd15_fbc_unet_ldm.safetensors": { | |
| "size": 1719167896, | |
| "hash": "97a662b8076504e0abad3b3a20b0e91d3312f2a5f19ffcef9059dab6d6679700", | |
| "muid": "4019c0f83d" | |
| }, | |
| "unet/iclight_sd15_fc_unet_ldm.safetensors": { | |
| "size": 1719144856, | |
| "hash": "9f91f1fc8ad2a2073c5a605fcd70cc70b2e7d2321b30aadca2a247d6490cd780", | |
| "muid": "f220618ed6" | |
| }, | |
| "unet/kolors_unet_fp16.safetensors": { | |
| "size": 5159140240, | |
| "hash": "425ff1dcbe3a70ac13d3afdd69bd4e3176b0c3260722527c80b210f11d2d966c", | |
| "muid": "9c8f088e4c" | |
| }, | |
| "llms/Helsinki-NLP/opus-mt-zh-en/pytorch_model.bin": { | |
| "size": 312087009, | |
| "hash": "9d8ceb91d103ef89400c9d9d62328b4858743cf8924878aee3b8afc594242ce0", | |
| "muid": "9d8ceb91d1" | |
| }, | |
| "llms/bert-base-uncased/model.safetensors": { | |
| "size": 440449768, | |
| "hash": "68d45e234eb4a928074dfd868cead0219ab85354cc53d20e772753c6bb9169d3", | |
| "muid": "9c02f497ee" | |
| }, | |
| "llms/nllb-200-distilled-600M/pytorch_model.bin": { | |
| "size": 2460457927, | |
| "hash": "c266c2cfd19758b6d09c1fc31ecdf1e485509035f6b51dfe84f1ada83eefcc42", | |
| "muid": "c266c2cfd1" | |
| }, | |
| "llms/superprompt-v1/model.safetensors": { | |
| "size": 307867048, | |
| "hash": "4f31e59c0582d4a74aac96ffb4ea9f5d64b268564ae5d1f68e8620dc940127d7", | |
| "muid": "ac31ee526b" | |
| }, | |
| "vae/ae.safetensors": { | |
| "size": 335304388, | |
| "hash": "afc8e28272cd15db3919bacdb6918ce9c1ed22e96cb12c4d5ed0fba823529e38", | |
| "muid": "ddec9c299f" | |
| }, | |
| "vae/ponyDiffusionV6XL_vae.safetensors": { | |
| "size": 334641162, | |
| "hash": "235745af8d86bf4a4c1b5b4f529868b37019a10f7c0b2e79ad0abca3a22bc6e1", | |
| "muid": "55f20a1016" | |
| }, | |
| "vae/sdxl_fp16.vae.safetensors": { | |
| "size": 167335342, | |
| "hash": "bcb60880a46b63dea58e9bc591abe15f8350bde47b405f9c38f4be70c6161e68", | |
| "muid": "345f7343ee" | |
| }, | |
| "checkpoints/Kolors-Inpainting.safetensors": { | |
| "size": 5159169040, | |
| "hash": "235db024626d7291e5d8af6776e8f49fa719c90221da9a54b553bb746101a787", | |
| "muid": "781857d59e" | |
| }, | |
| "controlnet/Kolors-ControlNet-Canny.safetensors": { | |
| "size": 2526129624, | |
| "hash": "ab34969b4ee57a182deb6e52e15d06c81c5285739caf4db2d8774135fd2b99e7", | |
| "muid": "0dec730f7e" | |
| }, | |
| "controlnet/Kolors-ControlNet-Depth.safetensors": { | |
| "size": 2526129624, | |
| "hash": "b2e9f9ff67c6c8e3b3fbe833f9596d9d16d456b1911633af9aeb4b80949ee60b", | |
| "muid": "0ad6e5c573" | |
| }, | |
| "controlnet/Kolors-ControlNet-Pose.safetensors": { | |
| "size": 2526129624, | |
| "hash": "2d21bbb821c903166c7c79f8a3435b51a39fd449cd227f74ac1d345bbc4eb153", | |
| "muid": "3fdfc617f9" | |
| }, | |
| "checkpoints/juggernautXL_v8Rundiffusion.safetensors": { | |
| "size": 7105348592, | |
| "hash": "aeb7e9e6897a1e58b10494bd989d001e3d4bc9b634633cd7b559838f612c2867", | |
| "muid": "f84d1c1e05d4" | |
| }, | |
| "checkpoints/juggernautXL_juggXIByRundiffusion.safetensors": { | |
| "size": 7105350536, | |
| "hash": "33e58e86686f6b386c526682b5da9228ead4f91d994abd4b053442dc5b42719e", | |
| "muid": "2f1dcc5762" | |
| }, | |
| "unet/Kolors-Inpainting.safetensors": { | |
| "size": 5159169040, | |
| "hash": "235db024626d7291e5d8af6776e8f49fa719c90221da9a54b553bb746101a787", | |
| "muid": "781857d59e" | |
| }, | |
| "controlnet/Kolors-ControlNet-Canny.safetensors": { | |
| "size": 2526129624, | |
| "hash": "ab34969b4ee57a182deb6e52e15d06c81c5285739caf4db2d8774135fd2b99e7", | |
| "muid": "0dec730f7e" | |
| }, | |
| "controlnet/Kolors-ControlNet-Depth.safetensors": { | |
| "size": 2526129624, | |
| "hash": "b2e9f9ff67c6c8e3b3fbe833f9596d9d16d456b1911633af9aeb4b80949ee60b", | |
| "muid": "0ad6e5c573" | |
| }, | |
| "controlnet/Kolors-ControlNet-Pose.safetensors": { | |
| "size": 2526129624, | |
| "hash": "2d21bbb821c903166c7c79f8a3435b51a39fd449cd227f74ac1d345bbc4eb153", | |
| "muid": "3fdfc617f9" | |
| }, | |
| } | |
| is_calc_hash = False # flag to print json update message | |
| def sync_model_info(downurls): | |
| print(f'downurls:{downurls}') | |
| keylist = [] | |
| return keylist | |
| class ModelsInfo: | |
| def __init__(self, models_info_path, path_map, scan_hash=False): | |
| self.scan_models_hash = scan_hash | |
| self.info_path = models_info_path | |
| self.path_map = path_map | |
| self.m_info = {} | |
| self.m_muid = {} | |
| self.m_file = {} | |
| self.load_model_info() | |
| self.refresh_from_path(scan_hash) | |
| def get_stat(self): | |
| return len(self.m_info) | |
| def load_model_info(self): | |
| if os.path.exists(self.info_path): | |
| try: | |
| with open(self.info_path, "r", encoding="utf-8") as json_file: | |
| self.m_info.update(json.load(json_file)) | |
| file_no_exists_list = [] | |
| for k in self.m_info.keys(): | |
| if self.m_info[k]['file']: | |
| model_files = self.m_info[k]['file'] | |
| exists_file_list = [] | |
| for file in model_files: | |
| file = file.replace("/", os.sep) | |
| if os.path.exists(file): | |
| if file in self.m_file and self.m_file[file]: | |
| self.m_file[file].append(k) | |
| else: | |
| self.m_file.update({file: [k]}) | |
| if file not in exists_file_list: | |
| exists_file_list.append(file) | |
| if len(exists_file_list) > 0: | |
| self.m_info[k]['file'] = exists_file_list | |
| else: | |
| file_no_exists_list.append(k) | |
| if k not in file_no_exists_list and self.m_info[k]['muid']: | |
| self.update_muid_map(self.m_info[k]['muid'], k) | |
| for k in file_no_exists_list: | |
| del self.m_info[k] | |
| #print(f'load m_info_key:{self.m_info.keys()}') | |
| except Exception as e: | |
| print(f'[ModelInfo] Load model info file {self.info_path} failed!, error:{e}') | |
| self.m_info = {} | |
| self.m_muid = {} | |
| self.m_file = {} | |
| def refresh_from_path(self, scan_hash=False): | |
| new_info_key = [] | |
| new_model_key = [] | |
| del_model_key = [] | |
| new_model_file = {} | |
| new_file_key = [] | |
| del_file_key = [] | |
| self.scan_models_hash = scan_hash | |
| for path in self.path_map.keys(): | |
| if self.path_map[path]: | |
| path_filenames = self.get_path_filenames(path) | |
| for (p, k) in path_filenames: | |
| model_key = f"{path}/{k.replace(os.sep, '/')}" | |
| file_path = os.path.join(p, k) | |
| if file_path not in new_file_key: | |
| new_file_key.append(file_path) | |
| if model_key in new_model_file: | |
| if file_path not in new_model_file[model_key]: | |
| new_model_file[model_key].append(file_path) | |
| else: | |
| new_model_file[model_key] = [file_path] | |
| if model_key not in new_info_key: | |
| new_info_key.append(model_key) | |
| if model_key not in self.m_info.keys(): | |
| new_model_key.append(model_key) | |
| if not utils.echo_off: | |
| print(f'[ModelInfo] new_model_key:{new_model_key}') | |
| for k in self.m_info.keys(): | |
| if k not in new_info_key: | |
| del_model_key.append(k) | |
| for f in self.m_file.keys(): | |
| if f not in new_file_key: | |
| del_file_key.append(f) | |
| if not utils.echo_off: | |
| print(f'[ModelInfo] del_model_key:{del_model_key}, del_file_key:{del_file_key}') | |
| for f in new_model_key: | |
| self.add_or_refresh_model(f, new_model_file[f]) | |
| for f in del_model_key: | |
| self.remove_model(f) | |
| for f in del_file_key: | |
| self.remove_file(f) | |
| self.save_model_info() | |
| def get_path_filenames(self, path): | |
| if path.isupper(): | |
| path_filenames = [] | |
| for f_path in self.path_map[path]: | |
| path_filenames += [(f_path, entry) for entry in os.listdir(f_path) if | |
| os.path.isdir(os.path.join(f_path, entry))] | |
| else: | |
| path_filenames = get_model_filenames(self.path_map[path]) | |
| return path_filenames | |
| def add_or_refresh_model(self, model_key, file_path_list, url=None): | |
| file_path_list_all = [] if model_key not in self.m_info else self.m_info[model_key]['file'] | |
| for file_path in file_path_list: | |
| if file_path not in file_path_list_all: | |
| file_path_list_all.append(file_path) | |
| url1 = '' if model_key not in self.m_info else self.m_info[model_key]['url'] | |
| url = url1 if url is None else url | |
| size, hash, muid = self.calculate_model_info(model_key, file_path_list[0]) | |
| self.m_info.update( | |
| {model_key: {'size': size, 'hash': hash, 'file': file_path_list_all, 'muid': muid, 'url': url}}) | |
| self.update_muid_map(muid, model_key) | |
| self.update_file_map(file_path_list_all, model_key) | |
| def remove_model(self, model_key): | |
| if self.m_info[model_key]['muid'] and self.m_info[model_key]['muid'] in self.m_muid: | |
| self.remove_muid_map(self.m_info[model_key]['muid'], model_key) | |
| if self.m_info[model_key]['file']: | |
| self.remove_file_map(self.m_info[model_key]['file'], model_key) | |
| del self.m_info[model_key] | |
| def remove_file(self, file_path): | |
| if file_path in self.m_file and self.m_file[file_path]: | |
| for model_key in self.m_file[file_path]: | |
| cata = model_key.split('/')[0] | |
| if cata.isupper(): | |
| continue | |
| if model_key in self.m_info and self.m_info[model_key]['file']: | |
| if file_path in self.m_info[model_key]['file']: | |
| self.m_info[model_key]['file'].remove(file_path) | |
| if len(self.m_info[model_key]['file']) == 0: | |
| self.remove_model(model_key) | |
| del self.m_file[file_path] | |
| def remove_file_map(self, file_paths, model_key): | |
| for file_path in file_paths: | |
| if file_path in self.m_file: | |
| if model_key in self.m_file[file_path]: | |
| self.m_file[file_path].remove(model_key) | |
| if len(self.m_file[file_path]) == 0: | |
| del self.m_file[file_path] | |
| def update_file_map(self, file_paths, model_key): | |
| for file_path in file_paths: | |
| if file_path in self.m_file: | |
| if model_key not in self.m_file[file_path]: | |
| self.m_file[file_path].append(model_key) | |
| else: | |
| self.m_file.update({file_path: [model_key]}) | |
| def update_muid_map(self, muid, model_key): | |
| if muid in self.m_muid and self.m_muid[muid]: | |
| if model_key not in self.m_muid[muid]: | |
| self.m_muid[muid].append(model_key) | |
| else: | |
| self.m_muid[muid] = [model_key] | |
| def remove_muid_map(self, muid, model_key): | |
| if muid in self.m_muid and self.m_muid[muid]: | |
| if model_key in self.m_muid[muid]: | |
| self.m_muid[muid].remove(model_key) | |
| if len(self.m_muid[muid]) == 0: | |
| del self.m_muid[muid] | |
| def calculate_model_info(self, model_key, file_path): | |
| global is_calc_hash | |
| if os.path.isdir(file_path): | |
| size = utils.get_size_subfolders(file_path) | |
| else: | |
| size = os.path.getsize(file_path) | |
| if model_key in default_models_info.keys() and size == default_models_info[model_key]["size"]: | |
| hash = default_models_info[model_key]["hash"] | |
| muid = default_models_info[model_key]["muid"] | |
| elif self.scan_models_hash: | |
| is_calc_hash = True | |
| print(f'[ModelInfo] Calculate hash for {file_path}') | |
| if os.path.isdir(file_path): | |
| hash = utils.calculate_sha256_subfolder(file_path) | |
| muid = hash[:10] | |
| else: | |
| hash = utils.sha256(file_path, length=None) | |
| _, file_extension = os.path.splitext(file_path) | |
| if file_extension == '.safetensors': | |
| print(f'[ModelInfo] Calculate addnet hash for {file_path}') | |
| muid = utils.sha256(file_path, use_addnet_hash=True) | |
| else: | |
| muid = hash[:10] | |
| else: | |
| hash = '' | |
| muid = '' | |
| return size, hash, muid | |
| def save_model_info(self): | |
| global is_calc_hash | |
| try: | |
| with open(self.info_path, "w", encoding="utf-8") as json_file: | |
| json.dump(self.m_info, json_file, indent=4) | |
| if is_calc_hash: | |
| print(f'[ModelInfo] Models info updated and saved to {self.info_path}') | |
| except PermissionError: | |
| print(f'[ModelInfo] Models info update and save failed: Permission denied, {self.info_path}') | |
| except json.JSONDecodeError: | |
| print(f'[ModelInfo] Models info update and save failed: JSON decode error, {self.info_path}') | |
| except Exception as e: | |
| print(f'[ModelInfo Models info update and save failed: {e}, {self.info_path}') | |
| def refresh_file(self, action, file_path, url=None): | |
| if action not in ['add', 'delete']: | |
| print(f'[ModelInfo] Invalid action: {action}. Action must be either "add" or "delete".') | |
| return | |
| if action == 'add': | |
| if not os.path.exists(file_path): | |
| print(f'[ModelInfo] The added file does not exist: {file_path}') | |
| return | |
| # Determine the catalog and model_name | |
| catalog = None | |
| max_match_length = 0 | |
| model_name = os.path.basename(file_path) | |
| for key, paths in self.path_map.items(): | |
| for path in paths: | |
| if file_path.startswith(path) and len(path) > max_match_length: | |
| catalog = key | |
| max_match_length = len(path) | |
| model_name = file_path[len(path) + 1:] | |
| if not catalog: | |
| print(f'[ModelInfo] The added file path {file_path} does not match any path in path_map.') | |
| return | |
| scan_hash = self.scan_models_hash | |
| self.scan_models_hash = True | |
| model_name = model_name.replace(os.sep, '/') | |
| model_key = f'{catalog}/{model_name}' | |
| self.add_or_refresh_model(model_key, [file_path], url) | |
| print(f'[ModelInfo] Added model {model_key} with file {file_path}') | |
| self.scan_models_hash = scan_hash | |
| elif action == 'delete': | |
| if file_path not in self.m_file: | |
| print(f'[ModelInfo] File not found in model info: {file_path}') | |
| return | |
| self.remove_file(file_path) | |
| print(f'[ModelInfo] Deleted model {model_key} with file {file_path}') | |
| self.save_model_info() | |
| def exists_model(self, catalog='', model_path='', muid=None): | |
| if muid and muid in self.m_muid: | |
| return True | |
| if catalog and model_path: | |
| model_path = model_path.replace('\\', '/').replace(os.sep, '/') | |
| model_key = f'{catalog}/{model_path}' | |
| if model_key in self.m_info: | |
| return True | |
| return False | |
| def exists_model_key(self, model_key): | |
| if model_key: | |
| cata = model_key.split('/')[0] | |
| model_path = model_key[len(cata) + 1:].replace('\\', '/').replace(os.sep, '/') | |
| model_key = f'{cata}/{model_path}' | |
| if model_key in self.m_info: | |
| return True | |
| return False | |
| def get_model_filepath(self, catalog='', model_path='', muid=None): | |
| if muid and muid in self.m_muid: | |
| model_key = self.m_muid[muid][0] | |
| file_paths = self.m_info[model_key]['file'] | |
| return file_paths[0] | |
| if catalog and model_path: | |
| model_path = model_path.replace('\\', '/').replace(os.sep, '/') | |
| model_key = f'{catalog}/{model_path}' | |
| if model_key in self.m_info: | |
| return self.m_info[model_key]['file'][0] | |
| return '' | |
| def get_model_names(self, catalog, filters=[], casesensitive=False, reverse=False): | |
| result = [] | |
| result_reverse = [] | |
| for f in self.m_info.keys(): | |
| cata = f.split('/')[0] | |
| m_path_or_file = f[len(cata) + 1:].replace('/', os.sep) | |
| if catalog and cata == catalog: | |
| result_reverse.append(m_path_or_file) | |
| if len(filters) > 0: | |
| for item in filters: | |
| if casesensitive: | |
| if item in m_path_or_file: | |
| result.append(m_path_or_file) | |
| result_reverse.pop() | |
| break | |
| else: | |
| if item.lower() in m_path_or_file.lower(): | |
| result.append(m_path_or_file) | |
| result_reverse.pop() | |
| break | |
| else: | |
| result.append(m_path_or_file) | |
| result_reverse.pop() | |
| if reverse: | |
| return sorted(result_reverse, key=str.casefold) | |
| return sorted(result, key=str.casefold) | |
| def get_model_info(self, catalog, model_path): | |
| if catalog and model_path: | |
| model_path = model_path.replace('\\', '/').replace(os.sep, '/') | |
| model_key = f'{catalog}/{model_path}' | |
| return self.get_model_key_info(model_key) | |
| def get_model_key_info(self, model_key): | |
| if model_key: | |
| cata = model_key.split('/')[0] | |
| model_path = model_key[len(cata) + 1:].replace('\\', '/').replace(os.sep, '/') | |
| model_key = f'{cata}/{model_path}' | |
| if model_key in self.m_info: | |
| return self.m_info[model_key] | |
| return None | |
| def get_file_muid(self, file_path): | |
| if file_path: | |
| if file_path not in self.m_file: | |
| self.refresh_file('add', file_path) | |
| model_key = self.m_file[file_path][0] | |
| muid = self.m_info[model_key]['muid'] | |
| if not muid: | |
| scan_hash = self.scan_models_hash | |
| self.scan_models_hash = True | |
| self.add_or_refresh_model(model_key, [file_path]) | |
| self.save_model_info() | |
| self.scan_models_hash = scan_hash | |
| muid = self.m_info[model_key]['muid'] | |
| return muid | |
| return '' | |
| def get_model_path_by_name(self, catalog, name, casesensitive=True, collection=False): | |
| if catalog and name: | |
| catalog = f'{catalog}/' | |
| if os.sep in name: | |
| name = name.replace(os.sep, '/') | |
| name1 = f'/{name}' | |
| if not casesensitive: | |
| name=name1.lower() | |
| catalog=catalog.lower() | |
| results = [] | |
| for f in self.m_info.keys(): | |
| if not casesensitive: | |
| f1=f.lower() | |
| else: | |
| f1=f | |
| if f1.startswith(catalog) and f1.endswith(name): | |
| cata = f.split('/')[0] | |
| model_path = f[len(cata) + 1:].replace('/', os.sep) | |
| if not collection: | |
| return model_path | |
| results.append(model_path) | |
| if collection: | |
| return results | |
| return '' | |
| def get_file_path_by_name(self, catalog, name, casesensitive=True, collection=False): | |
| if catalog and name: | |
| cata = f'{catalog}/' | |
| if os.sep in name: | |
| name = name.replace(os.sep, '/') | |
| name1 = f'/{name}' | |
| if not casesensitive: | |
| name1=name1.lower() | |
| cata=cata.lower() | |
| results = [] | |
| for f in self.m_info.keys(): | |
| if not casesensitive: | |
| f1=f.lower() | |
| else: | |
| f1=f | |
| if f1.startswith(cata) and f1.endswith(name1): | |
| file_paths = self.m_info[f]['file'] | |
| if not collection: | |
| return file_paths[0] | |
| results.append(file_paths[0]) | |
| if collection and len(results)>0: | |
| return results | |
| return os.path.join(self.path_map[catalog][0], name.replace('/', os.sep)) | |
| return '' | |
| def get_model_filenames(folder_paths, extensions=None, name_filter=None, variation=False): | |
| if extensions is None: | |
| extensions = ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch', '.gguf'] | |
| files = [] | |
| for folder in folder_paths: | |
| files += get_files_from_folder(folder, extensions, name_filter, variation) | |
| return files | |
| folder_variation = {} | |
| def get_files_from_folder(folder_path, extensions=None, name_filter=None, variation=False): | |
| global folder_variation | |
| if not os.path.isdir(folder_path): | |
| raise ValueError("Folder path is not a valid directory.") | |
| filenames = [] | |
| for root, dirs, files in os.walk(folder_path, topdown=False): | |
| relative_path = os.path.relpath(root, folder_path) | |
| if relative_path == ".": | |
| relative_path = "" | |
| for filename in sorted(files, key=lambda s: s.casefold()): | |
| _, file_extension = os.path.splitext(filename) | |
| if (extensions is None or file_extension.lower() in extensions) and ( | |
| name_filter is None or name_filter in _): | |
| path = os.path.join(relative_path, filename) | |
| if variation: | |
| mtime = int(os.path.getmtime(os.path.join(root, filename))) | |
| if folder_path not in folder_variation or path not in folder_variation[folder_path] or mtime > \ | |
| folder_variation[folder_path][path]: | |
| if folder_path not in folder_variation: | |
| folder_variation.update({folder_path: {path: mtime}}) | |
| else: | |
| folder_variation[folder_path].update({path: mtime}) | |
| filenames.append((folder_path, path)) | |
| else: | |
| filenames.append((folder_path, path)) | |
| return filenames | |