Allow plain text files as Content to use. (#1)

Joo200 · mcmonkey4eva · web-flow · commit 492563ff8635 · 2023-11-03T06:05:05.000-07:00
* Allow Plain Text

* cleanup (see pr comment)

* missed format fix

---------

Co-authored-by: Alex "mcmonkey" Goodwin &lt;git_commits@alexgoodwin.dev&gt;
diff --git a/SharpDenizenTools/MetaHandlers/MetaDocsLoader.cs b/SharpDenizenTools/MetaHandlers/MetaDocsLoader.cs
@@ -70,7 +70,7 @@ public static MetaDocs DownloadAll(string[] sources)
                 docs.LoadErrors.Add($"Internal exception while reading extra data - {ex.GetType().FullName} ... see bot console for details.");
                 Console.Error.WriteLine($"Error: {ex}");
             }
-            ConcurrentDictionary<string, ZipArchive> zips = new();
+            ConcurrentDictionary<string, object> files = new();
             List<ManualResetEvent> resets = new();
             foreach (string src in sources)
             {
@@ -80,13 +80,21 @@ public static MetaDocs DownloadAll(string[] sources)
                 {
                     try
                     {
-                        ZipArchive zip = DownloadZip(webClient, src);
-                        zips[src] = zip;
+                        byte[] data = DownloadData(webClient, src);
+                        // Note: backup check based on PKWARE zip header (PK followed by 2 control bytes, usually 0x03 0x04)
+                        if (src.EndsWith(".zip") || (data[0] == 0x50 && data[1] == 0x4b && data[1] < 0x20 && data[2] < 0x20))
+                        {
+                            files[src] = new ZipArchive(new MemoryStream(data));
+                        }
+                        else
+                        {
+                            files[src] = StringConversionHelper.UTF8Encoding.GetString(data, 0, data.Length);
+                        }
                     }
                     catch (Exception ex)
                     {
-                        Console.Error.WriteLine($"Zip download exception {ex}");
-                        docs.LoadErrors.Add($"Zip download error: {ex.GetType().Name}: {ex.Message}");
+                        Console.Error.WriteLine($"Source download exception {ex}");
+                        docs.LoadErrors.Add($"Source download error: {ex.GetType().Name}: {ex.Message}");
                     }
                     evt.Set();
                 });
@@ -95,11 +103,21 @@ public static MetaDocs DownloadAll(string[] sources)
             {
                 evt.WaitOne();
             }
-            foreach (string src in sources)
+            foreach ((string src, object file) in files)
             {
                 try
                 {
-                    (int, string, string)[] fullLines = ReadLines(zips[src]);
+                    (int, string, string)[] fullLines;
+                    if (file is ZipArchive zip)
+                    {
+                        fullLines = ReadLines(zip);
+                    }
+                    else
+                    {
+                        List<(int, string, string)> lines = new();
+                        SeparateDataLines(lines, src, (file as string).Split('\n'));
+                        fullLines = lines.ToArray();
+                    }
                     LoadDataFromLines(docs, src, fullLines);
                 }
                 catch (Exception ex)
@@ -229,7 +247,7 @@ public static void ReadGuides(MetaDocs docs, HttpClient client)
         }
 
         /// <summary>Download a zip file from a URL.</summary>
-        public static ZipArchive DownloadZip(HttpClient webClient, string url)
+        public static byte[] DownloadData(HttpClient webClient, string url)
         {
             byte[] zipDataBytes;
             if (AlternateZipSourcer != null)
@@ -240,8 +258,7 @@ public static ZipArchive DownloadZip(HttpClient webClient, string url)
             {
                 zipDataBytes = webClient.GetByteArrayAsync(url).Result;
             }
-            MemoryStream zipDataStream = new(zipDataBytes);
-            return new ZipArchive(zipDataStream);
+            return zipDataBytes;
         }
 
         /// <summary>Read lines of meta docs from Java files in a zip.</summary>

Original file line number	Diff line number	Diff line change
`@@ -70,7 +70,7 @@ public static MetaDocs DownloadAll(string[] sources)`
`70`	`70`	`docs.LoadErrors.Add($"Internal exception while reading extra data - {ex.GetType().FullName} ... see bot console for details.");`
`71`	`71`	`Console.Error.WriteLine($"Error: {ex}");`
`72`	`72`	`}`
`73`		`- ConcurrentDictionary<string, ZipArchive> zips = new();`
	`73`	`+ ConcurrentDictionary<string, object> files = new();`
`74`	`74`	`List<ManualResetEvent> resets = new();`
`75`	`75`	`foreach (string src in sources)`
`76`	`76`	`{`
`@@ -80,13 +80,21 @@ public static MetaDocs DownloadAll(string[] sources)`
`80`	`80`	`{`
`81`	`81`	`try`
`82`	`82`	`{`
`83`		`- ZipArchive zip = DownloadZip(webClient, src);`
`84`		`- zips[src] = zip;`
	`83`	`+ byte[] data = DownloadData(webClient, src);`
	`84`	`+ // Note: backup check based on PKWARE zip header (PK followed by 2 control bytes, usually 0x03 0x04)`
	`85`	`+ if (src.EndsWith(".zip") \|\| (data[0] == 0x50 && data[1] == 0x4b && data[1] < 0x20 && data[2] < 0x20))`
	`86`	`+ {`
	`87`	`+ files[src] = new ZipArchive(new MemoryStream(data));`
	`88`	`+ }`
	`89`	`+ else`
	`90`	`+ {`
	`91`	`+ files[src] = StringConversionHelper.UTF8Encoding.GetString(data, 0, data.Length);`
	`92`	`+ }`
`85`	`93`	`}`
`86`	`94`	`catch (Exception ex)`
`87`	`95`	`{`
`88`		`- Console.Error.WriteLine($"Zip download exception {ex}");`
`89`		`- docs.LoadErrors.Add($"Zip download error: {ex.GetType().Name}: {ex.Message}");`
	`96`	`+ Console.Error.WriteLine($"Source download exception {ex}");`
	`97`	`+ docs.LoadErrors.Add($"Source download error: {ex.GetType().Name}: {ex.Message}");`
`90`	`98`	`}`
`91`	`99`	`evt.Set();`
`92`	`100`	`});`
`@@ -95,11 +103,21 @@ public static MetaDocs DownloadAll(string[] sources)`
`95`	`103`	`{`
`96`	`104`	`evt.WaitOne();`
`97`	`105`	`}`
`98`		`- foreach (string src in sources)`
	`106`	`+ foreach ((string src, object file) in files)`
`99`	`107`	`{`
`100`	`108`	`try`
`101`	`109`	`{`
`102`		`- (int, string, string)[] fullLines = ReadLines(zips[src]);`
	`110`	`+ (int, string, string)[] fullLines;`
	`111`	`+ if (file is ZipArchive zip)`
	`112`	`+ {`
	`113`	`+ fullLines = ReadLines(zip);`
	`114`	`+ }`
	`115`	`+ else`
	`116`	`+ {`
	`117`	`+ List<(int, string, string)> lines = new();`
	`118`	`+ SeparateDataLines(lines, src, (file as string).Split('\n'));`
	`119`	`+ fullLines = lines.ToArray();`
	`120`	`+ }`
`103`	`121`	`LoadDataFromLines(docs, src, fullLines);`
`104`	`122`	`}`
`105`	`123`	`catch (Exception ex)`
`@@ -229,7 +247,7 @@ public static void ReadGuides(MetaDocs docs, HttpClient client)`
`229`	`247`	`}`
`230`	`248`
`231`	`249`	`/// <summary>Download a zip file from a URL.</summary>`
`232`		`- public static ZipArchive DownloadZip(HttpClient webClient, string url)`
	`250`	`+ public static byte[] DownloadData(HttpClient webClient, string url)`
`233`	`251`	`{`
`234`	`252`	`byte[] zipDataBytes;`
`235`	`253`	`if (AlternateZipSourcer != null)`
`@@ -240,8 +258,7 @@ public static ZipArchive DownloadZip(HttpClient webClient, string url)`
`240`	`258`	`{`
`241`	`259`	`zipDataBytes = webClient.GetByteArrayAsync(url).Result;`
`242`	`260`	`}`
`243`		`- MemoryStream zipDataStream = new(zipDataBytes);`
`244`		`- return new ZipArchive(zipDataStream);`
	`261`	`+ return zipDataBytes;`
`245`	`262`	`}`
`246`	`263`
`247`	`264`	`/// <summary>Read lines of meta docs from Java files in a zip.</summary>`