| """ | |
| Format: directly use cid as the built-in index position; if aid cannot be put in, use the dictionary to search, there are only 50,000 of them anyway | |
| """ | |
| import faiss, numpy as np, os | |
| # ############ If it is an original feature, you need to write save first | |
| inp_root = r"E:\codes\py39\dataset\mi\2-co256" | |
| npys = [] | |
| for name in sorted(list(os.listdir(inp_root))): | |
| phone = np.load("%s/%s" % (inp_root, name)) | |
| npys.append(phone) | |
| big_npy = np.concatenate(npys, 0) | |
| print(big_npy.shape) # (6196072, 192)#fp32#4.43G | |
| np.save("infer/big_src_feature_mi.npy", big_npy) | |
| ##################train+add | |
| # big_npy=np.load("/bili-coeus/jupyter/jupyterhub-liujing04/vits_ch/inference_f0/big_src_feature_mi.npy") | |
| print(big_npy.shape) | |
| index = faiss.index_factory(256, "IVF512,Flat") # mi | |
| print("training") | |
| index_ivf = faiss.extract_index_ivf(index) # | |
| index_ivf.nprobe = 9 | |
| index.train(big_npy) | |
| faiss.write_index(index, "infer/trained_IVF512_Flat_mi_baseline_src_feat.index") | |
| print("adding") | |
| index.add(big_npy) | |
| faiss.write_index(index, "infer/added_IVF512_Flat_mi_baseline_src_feat.index") | |
| """ | |
| Size (all FP32) | |
| big_src_feature 2.95G | |
| (3098036, 256) | |
| big_emb 4.43G | |
| (6196072, 192) | |
| The double of big_emb is because the feature needs to be repeated and then pitched | |
| """ | |