update script

civitai.info.js

@@ -1,10 +1,21 @@
-/*  This script exists to merge all .info files into one array.
+/* 
+ *  This script exists to merge all .info files into one array.
  *  unfortunately, civitai is storing some of them as LFS objects
  *  so the script outputs a list of errors along with scripts to fix them (delfiles.sh, replacefiles.sh)
  *  others files are blank, so just ignore the errors the second time, they're files that aren't on civitai
  *  now we get tot he important part: it parses all files, and dumps all of them into an array in civitai.info.json for further processing
  *  Run once, run delfiles.bat or delfiles.sh to get rid of the bad ones, then run replacefiles.sh to fetch them from civitai.
+ * 
+ *  the reason I chose to output scripts (and use two of them) rather than doing it internally is two-fold:
+ *  1. I can inspect delfiles.sh being deleted before running it
+ *  2. I can rewrite the replacefiles.sh with a script to use a download manager (aria2c) when it's large. (this script is intended to be used in colab)
  */
+
+// change this to true if you want the full contents of every .info file, otherwise we're deleting a bunch of stuff I don't care about
+// keep in mind this changes the format of the file (I've replaced the image object with an array of urls that have no metadata)
+// so any scripts I write won't work with it anymore. This reduced the size of the file by nearly 80%...The original data is still in the original files.
+var fullFiles = false; 
+
 var path = require('path'),
 fs = require('fs');
 
@@ -30,26 +41,51 @@ function fromDir(startPath, filter, arr) {
     return arr;
 };
 
-var addons=[];
-var errors=[];
+var addons=[]; var errors=[]; errors2=[];
 var push=function(file){
 	try{
 		var addon=JSON.parse(fs.readFileSync(file));
-		addon.mirror_path=file; // Add the location we found the file.
-		addons.push(addon)
 	} catch(err) {
-		console.log(file);
-		console.error(err)
+		console.error(file + " is probably an LFS pointer. Will be added to delfiles and replacefiles scripts!");
 		errors.push(file);
+		return;
+	}
+	if(Object.keys(addon).length==0) {
+		console.error(file + " is an empty file! Will be added to delfiles script! Assuming it's not a mistake, it will *not* be replaced!");
+		errors2.push(file);
+		return;
 	}
+	addon.mirror_path=file; // Add the location we found the file.
+
+	// We're going to clean information idc about. Change `false` to `true` at the top if you want all data.
+	if(fullFiles == true) {
+		addons.push(addon);
+		return;
+	}
+	delete addon.model.poi;
+	'createdAt,updatedAt,earlyAccessTimeFrame,downloadUrl,description'.split(',').forEach(key=>{delete addon[key]});
+	for(file=addon.files.length-1;file>=0;file--) {
+		if(addon.files[file].type == 'Training Data' || addon.files[file].type == 'Config') {
+			addon.files.splice(file,1);
+		} else {
+			'sizeKB,format,pickleScanResult,pickleScanMessage,virusScanResult,scannedAt,hashes,primary,downloadUrl'.split(',').forEach(key=>{
+				delete addon.files[file][key];
+			});
+		}
+	};
+	var imgs=addon.images;
+	addon.images=[];
+	imgs.forEach(img=>{addon.images.push(img.url.replace('https://imagecache.civitai.com/',''))});
+	addons.push(addon);
 };
 
 var files=fromDir('.', '.civitai.info');
-files.map(e=>{push(e);});
-
+files.forEach(file=>{push(file)});
+//push(files[0]);
+var errs=[...errors,...errors2].sort();
 fs.writeFileSync('civitai.info.json',JSON.stringify(addons))
-fs.writeFileSync('errors.json',JSON.stringify(errors))
+fs.writeFileSync('errors.json',JSON.stringify(errs))
 //console.error("Error with the following files:",errors);
-fs.writeFileSync('delfiles.bat','@echo off\n\n'+errors.map(e=>{return "del \""+ e + "\""}).join('\n'))
-fs.writeFileSync('delfiles.sh',errors.map(e=>{return "rm \""+ e + "\""}).join('\n').split('\\').join('/'))
-fs.writeFileSync('replacefiles.sh',errors.map(e=>{return "wget \"https://huggingface.co/anonderpling/civitai_mirror/resolve/main/"+(e.split(" ").join("%23")+"\" -O \""+e+"\"")}).join('\n').split('\\').join('\/'))
+fs.writeFileSync('delfiles.bat','@echo off\n\n'+errs.map(e=>{return "del \""+ e + "\""}).join('\n'))
+fs.writeFileSync('delfiles.sh',errs.map(e=>{return "rm \""+ e + "\""}).join('\n').split('\\').join('/'))
+fs.writeFileSync('replacefiles.sh',errors.map(e=>{return "wget \"https://huggingface.co/anonderpling/civitai_mirror/resolve/main/"+(e.split(" ").join("%23")+"\" -O \""+e+"\"")}).join('\n').split('\\').join('\/'))