Spaces:
Running
Running
File size: 3,015 Bytes
4384839 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import * as hub from "@huggingface/hub";
import type { RepoDesignation } from "@huggingface/hub";
import type { ContentSource } from "@huggingface/hub";
type FileArray = Array<URL | File | { path: string; content: ContentSource }>;
/**
* Uploads a leRobot dataset to huggingface
*/
export class LeRobotHFUploader extends EventTarget {
private _repoDesignation: RepoDesignation;
private _uploaded : boolean;
private _created_repo : boolean;
constructor(username: string, repoName: string) {
super();
this._repoDesignation = {
name : `${username}/${repoName}`,
type : "dataset"
};
this._uploaded = false;
this._created_repo = false;
}
/**
* Returns whether the repository has been successfully created
*/
get createdRepo(): boolean {
return this._created_repo;
}
get uploaded() : boolean {
return this._uploaded;
}
/**
* Uploads the dataset to huggingface
*
* A referenceId is used to be able to track progress of the upload,
* this provides a progressEvent from huggingface.js (see : https://github.com/huggingface/huggingface.js/blob/main/packages/hub/README.md#usage)
*
* both this and huggingface.js have pretty bad documentation, some exploration will be required (sorry!, I have university, and work and I already feel guilty procastinating those to to write this)
*
* @param dataset The dataset to upload
* @param accessToken The access token for huggingface
* @param referenceId The reference id for the upload, to track it (optional)
*/
async createRepoAndUploadFiles(files : FileArray, accessToken : string, referenceId : string = "") {
await hub.createRepo({
repo: this._repoDesignation,
accessToken: accessToken,
license: "mit"
});
this._created_repo = true;
this.dispatchEvent(new CustomEvent("repoCreated", { detail: this._repoDesignation }));
const uploadPromises : Promise<void>[] = [];
uploadPromises.push(this.uploadFilesWithProgress(files, accessToken, referenceId));
await Promise.all(uploadPromises);
}
/**
* Uploads files to huggingface with progress events
*
* @param files The files to upload
* @param accessToken The access token for huggingface
* @param referenceId The reference id for the upload, to track it (optional)
*/
async uploadFilesWithProgress(files : FileArray, accessToken : string, referenceId : string = "") {
for await (const progressEvent of hub.uploadFilesWithProgress({
repo: this._repoDesignation,
accessToken: accessToken,
files: files,
})) {
this.dispatchEvent(new CustomEvent("progress", { detail: {
progressEvent,
repoDesignation: this._repoDesignation,
referenceId
} }));
}
}
} |