From 82d880dbff0d1695513c462aaa559b1dd4becdd9 Mon Sep 17 00:00:00 2001
From: Romein van Buren <romein@vburen.nl>
Date: Sun, 28 May 2023 21:54:40 +0200
Subject: [PATCH] Improvememts to export tool and logging

---
 .../collection/components/export.svelte       |  24 +-
 frontend/wailsjs/go/models.ts                 |   2 +
 frontend/wailsjs/go/ui/UI.d.ts                |   2 +-
 frontend/wailsjs/go/ui/UI.js                  |   4 +-
 internal/app/app.go                           |  10 +-
 internal/app/collection_find_export.go        | 217 ++++++++++++++----
 internal/logger.go                            |  80 +++++++
 internal/ui/dialogs.go                        |   2 +-
 main.go                                       |   4 +-
 9 files changed, 283 insertions(+), 62 deletions(-)
 create mode 100644 internal/logger.go

diff --git a/frontend/src/organisms/connection/collection/components/export.svelte b/frontend/src/organisms/connection/collection/components/export.svelte
index 2cbee99..8e61778 100644
--- a/frontend/src/organisms/connection/collection/components/export.svelte
+++ b/frontend/src/organisms/connection/collection/components/export.svelte
@@ -1,8 +1,8 @@
 <script>
   import Icon from '$components/icon.svelte';
   import Modal from '$components/modal.svelte';
-  import { startProgress } from '$lib/progress';
   import views from '$lib/stores/views';
+  import { PerformFindExport } from '$wails/go/app/App';
   import { createEventDispatcher } from 'svelte';
 
   export let info;
@@ -11,16 +11,21 @@
   const dispatch = createEventDispatcher();
   let viewKey = collection.viewKey;
   $: viewKey = collection.viewKey;
+  $: if (info) {
+    info.viewKey = viewKey;
+  }
 
   async function performExport() {
-    const progress = startProgress('Performing export…');
     info.view = $views[viewKey];
-    //...
-    progress.end();
+    const success = await PerformFindExport(collection.hostKey, collection.dbKey, collection.key, JSON.stringify(info));
+
+    if (success) {
+      info = undefined;
+    }
   }
 </script>
 
-<Modal bind:show={info} title="Export results" width="400px">
+<Modal bind:show={info} title="Export results" width="450px">
   <form on:submit|preventDefault={performExport}>
     <label class="field">
       <span class="label">Export</span>
@@ -30,14 +35,16 @@
         <option value="querylimitskip">all records matching query, considering limit and skip</option>
       </select>
     </label>
+
     <label class="field">
       <span class="label">Format</span>
       <select bind:value={info.format}>
         <option value="jsonarray">JSON array</option>
-        <option value="jsonnewline">JSON: newline-separated objects</option>
+        <option value="ndjson">Newline delimited JSON</option>
         <option value="csv">CSV</option>
       </select>
     </label>
+
     <label class="field">
       <span class="label">View to use</span>
       <select bind:value={viewKey}>
@@ -49,6 +56,11 @@
         <Icon name="cog" />
       </button>
     </label>
+
+    <button class="btn" type="submit">
+      <Icon name="play" />
+      Start export
+    </button>
   </form>
 </Modal>
 
diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts
index 12a250e..d10e2e5 100755
--- a/frontend/wailsjs/go/models.ts
+++ b/frontend/wailsjs/go/models.ts
@@ -9,6 +9,7 @@ export namespace app {
 	    homeDirectory: string;
 	    dataDirectory: string;
 	    logDirectory: string;
+	    downloadDirectory: string;
 	
 	    static createFrom(source: any = {}) {
 	        return new EnvironmentInfo(source);
@@ -24,6 +25,7 @@ export namespace app {
 	        this.homeDirectory = source["homeDirectory"];
 	        this.dataDirectory = source["dataDirectory"];
 	        this.logDirectory = source["logDirectory"];
+	        this.downloadDirectory = source["downloadDirectory"];
 	    }
 	}
 	export class QueryResult {
diff --git a/frontend/wailsjs/go/ui/UI.d.ts b/frontend/wailsjs/go/ui/UI.d.ts
index 4fa613d..13da8e8 100755
--- a/frontend/wailsjs/go/ui/UI.d.ts
+++ b/frontend/wailsjs/go/ui/UI.d.ts
@@ -6,7 +6,7 @@ export function Beep():Promise<void>;
 
 export function EnterText(arg1:string,arg2:string,arg3:string):Promise<string>;
 
-export function OpenDirectory(arg1:string,arg2:string):Promise<string>;
+export function OpenDirectory(arg1:string):Promise<string>;
 
 export function Reveal(arg1:string):Promise<void>;
 
diff --git a/frontend/wailsjs/go/ui/UI.js b/frontend/wailsjs/go/ui/UI.js
index 6ee9f34..ce98890 100755
--- a/frontend/wailsjs/go/ui/UI.js
+++ b/frontend/wailsjs/go/ui/UI.js
@@ -10,8 +10,8 @@ export function EnterText(arg1, arg2, arg3) {
   return window['go']['ui']['UI']['EnterText'](arg1, arg2, arg3);
 }
 
-export function OpenDirectory(arg1, arg2) {
-  return window['go']['ui']['UI']['OpenDirectory'](arg1, arg2);
+export function OpenDirectory(arg1) {
+  return window['go']['ui']['UI']['OpenDirectory'](arg1);
 }
 
 export function Reveal(arg1) {
diff --git a/internal/app/app.go b/internal/app/app.go
index 75dac69..afc3241 100644
--- a/internal/app/app.go
+++ b/internal/app/app.go
@@ -22,9 +22,10 @@ type EnvironmentInfo struct {
 	HasMongoExport bool `json:"hasMongoExport"`
 	HasMongoDump   bool `json:"hasMongoDump"`
 
-	HomeDirectory string `json:"homeDirectory"`
-	DataDirectory string `json:"dataDirectory"`
-	LogDirectory  string `json:"logDirectory"`
+	HomeDirectory     string `json:"homeDirectory"`
+	DataDirectory     string `json:"dataDirectory"`
+	LogDirectory      string `json:"logDirectory"`
+	DownloadDirectory string `json:"downloadDirectory"`
 }
 
 type App struct {
@@ -54,12 +55,15 @@ func NewApp() *App {
 	case "windows":
 		a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/AppData/Local/Rolens")
 		a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/AppData/Local/Rolens/Logs")
+		a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
 	case "darwin":
 		a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/Library/Application Support/Rolens")
 		a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/Library/Logs/Rolens")
+		a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
 	case "linux":
 		a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/.config/rolens")
 		a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/.config/rolens/logs")
+		a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
 	default:
 		panic(errors.New("unsupported platform"))
 	}
diff --git a/internal/app/collection_find_export.go b/internal/app/collection_find_export.go
index 88996d5..01a5fce 100644
--- a/internal/app/collection_find_export.go
+++ b/internal/app/collection_find_export.go
@@ -9,6 +9,7 @@ import (
 	"github.com/ncruces/zenity"
 	"github.com/wailsapp/wails/v2/pkg/runtime"
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
@@ -20,9 +21,9 @@ const (
 	ExportContentsQuery          ExportContents = "query"
 	ExportContentsQueryLimitSkip ExportContents = "querylimitskip"
 
-	ExportFormatJsonArray   ExportFormat = "jsonarray"
-	ExportFormatJsonNewline ExportFormat = "jsonnewline"
-	ExportFormatCsv         ExportFormat = "csv"
+	ExportFormatJsonArray ExportFormat = "jsonarray"
+	ExportFormatNdJson    ExportFormat = "ndjson"
+	ExportFormatCsv       ExportFormat = "csv"
 )
 
 type ExportSettings struct {
@@ -30,41 +31,101 @@ type ExportSettings struct {
 	Format    ExportFormat   `json:"format"`
 	ViewKey   string         `json:"viewKey"`
 	QueryJson string         `json:"query"`
-	Limit     int64          `json:"limit"`
-	Skip      int64          `json:"skip"`
+	Limit     uint           `json:"limit"`
+	Skip      uint           `json:"skip"`
 	OutFile   string         `json:"outfile"`
 }
 
+func getptr[T any](v T) *T {
+	return &v
+}
+
 func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bool {
+	runtime.LogInfof(a.ctx, "Export started for %s/%s/%s. Settings: %s", hostKey, dbKey, collKey, settingsJson)
+
 	var settings ExportSettings
 	if err := json.Unmarshal([]byte(settingsJson), &settings); err != nil {
-		runtime.LogWarning(a.ctx, "Could not parse export settings:")
-		runtime.LogWarning(a.ctx, err.Error())
+		runtime.LogWarningf(a.ctx, "Could not parse export settings: %s", err.Error())
 		zenity.Error(err.Error(), zenity.Title("Couldn't parse export settings!"), zenity.ErrorIcon)
 		return false
 	}
 
-	if _, err := os.Stat(settings.OutFile); err == nil {
-		zenity.Error(fmt.Sprintf("File %s already exists, export aborted.", settings.OutFile), zenity.ErrorIcon)
-		return false
+	switch settings.Contents {
+	case ExportContentsAll:
+		settings.QueryJson = "{}"
+		settings.Limit = 0
+		settings.Skip = 0
+	case ExportContentsQuery:
+		settings.Limit = 0
+		settings.Skip = 0
+	case ExportContentsQueryLimitSkip:
 	}
 
 	views, err := a.Views()
 	if err != nil {
+		runtime.LogWarningf(a.ctx, "Export: error while retrieving view: %s", err.Error())
 		return false
 	}
 
 	view, found := views[settings.ViewKey]
 	if !found {
 		zenity.Error(fmt.Sprintf("View %s is not known", settings.ViewKey), zenity.ErrorIcon)
+		runtime.LogDebugf(a.ctx, "Export: unknown view %s", settings.ViewKey)
+		return false
+	}
+
+	var fileFilter runtime.FileFilter
+	defaultFilename := ""
+
+	switch settings.Format {
+	case ExportFormatCsv:
+		defaultFilename = "export.csv"
+		fileFilter = runtime.FileFilter{
+			DisplayName: "CSV files (*.csv)",
+			Pattern:     "*.csv",
+		}
+	case ExportFormatJsonArray:
+		defaultFilename = "export.json"
+		fileFilter = runtime.FileFilter{
+			DisplayName: "JSON files (*.json)",
+			Pattern:     "*.json",
+		}
+	case ExportFormatNdJson:
+		defaultFilename = "export.ndjson"
+		fileFilter = runtime.FileFilter{
+			DisplayName: "Newline delimited JSON files (*.ndjson)",
+			Pattern:     "*.ndjson",
+		}
+	}
+
+	settings.OutFile, err = runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
+		Title:                "Choose export destination",
+		DefaultDirectory:     a.Env.DownloadDirectory,
+		CanCreateDirectories: true,
+		DefaultFilename:      defaultFilename,
+		Filters:              []runtime.FileFilter{fileFilter},
+	})
+	if err != nil {
+		zenity.Error("An error occured while choosing the export destination", zenity.ErrorIcon)
+		runtime.LogWarningf(a.ctx, "Export: error while choosing export destination: %s", err.Error())
+		return false
+	}
+	if settings.OutFile == "" {
+		zenity.Error("You must specify an export destination.", zenity.ErrorIcon)
+		runtime.LogDebug(a.ctx, "Export: no destination specified")
+		return false
+	}
+
+	if _, err := os.Stat(settings.OutFile); err == nil {
+		zenity.Error(fmt.Sprintf("File %s already exists, export aborted.", settings.OutFile), zenity.ErrorIcon)
+		runtime.LogDebugf(a.ctx, "Export: destination %s already exists", settings.OutFile)
 		return false
 	}
 
 	var query bson.M
 	if settings.Contents != ExportContentsAll {
 		if err = bson.UnmarshalExtJSON([]byte(settings.QueryJson), true, &query); err != nil {
-			runtime.LogInfo(a.ctx, "Invalid find query (exporting):")
-			runtime.LogInfo(a.ctx, err.Error())
+			runtime.LogDebugf(a.ctx, "Invalid find query (exporting): %s", settings.QueryJson)
 			zenity.Error(err.Error(), zenity.Title("Invalid query"), zenity.ErrorIcon)
 			return false
 		}
@@ -76,22 +137,32 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
 	}
 	defer close()
 
-	projection := bson.M{}
+	pgr, _ := zenity.Progress(zenity.Title("Performing export…"))
 
+	projection := bson.M{}
 	if settings.ViewKey != "list" {
 		for _, col := range view.Columns {
 			projection[col.Key] = ""
 		}
 	}
 
+	var count uint = 0
+	if settings.Limit != 0 {
+		count = uint(settings.Limit)
+	} else {
+		c, _ := client.Database(dbKey).Collection(collKey).CountDocuments(ctx, query, &options.CountOptions{
+			Skip: getptr(int64(settings.Skip)),
+		})
+		count = uint(c)
+	}
+
 	cur, err := client.Database(dbKey).Collection(collKey).Find(ctx, query, &options.FindOptions{
-		Skip:       &settings.Skip,
-		Limit:      &settings.Limit,
+		Skip:       getptr(int64(settings.Skip)),
+		Limit:      getptr(int64(settings.Limit)),
 		Projection: projection,
 	})
 	if err != nil {
-		runtime.LogInfo(a.ctx, "Unable to get cursor while exporting:")
-		runtime.LogInfo(a.ctx, err.Error())
+		runtime.LogInfof(a.ctx, "Export: unable to get cursor while exporting: %s", err.Error())
 		zenity.Error(err.Error(), zenity.Title("Unable to get cursor"), zenity.ErrorIcon)
 		return false
 	}
@@ -99,74 +170,119 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
 	file, err := os.OpenFile(settings.OutFile, os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {
 		zenity.Error(fmt.Sprintf(err.Error(), zenity.Title("Error while opening file"), settings.OutFile), zenity.ErrorIcon)
+		runtime.LogDebugf(a.ctx, "Export: unable to open file %s", settings.OutFile)
 		return false
 	}
 	defer file.Close()
 
 	var csvWriter *csv.Writer
-	var csvColumnKeys []any
+	var csvColumnKeys []string
+	var index uint = 0
 
 	switch settings.Format {
 	case ExportFormatJsonArray:
-		file.WriteString("[\n")
+		file.WriteString("[")
+
 	case ExportFormatCsv:
 		csvWriter = csv.NewWriter(file)
 	}
 
-	index := -1
 	for cur.Next(ctx) {
-		index++
-		var item map[any]interface{}
-
-		if err := bson.Unmarshal(cur.Current, &item); err != nil {
-			runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to unmarshal item %d while exporting", index))
-			runtime.LogInfo(a.ctx, err.Error())
-			zenity.Error(err.Error(), zenity.Title("Unable to unmarshal item %d"), zenity.ErrorIcon)
-			continue
-		}
-
 		switch settings.Format {
 		case ExportFormatCsv:
+			els, err := cur.Current.Elements()
+			if err != nil {
+				zenity.Error(err.Error(), zenity.Title("BSON invalid"), zenity.ErrorIcon)
+			}
+
 			csvItem := make([]string, 0)
 
 			switch settings.ViewKey {
 			case "list":
 				if csvColumnKeys == nil {
-					csvColumnKeys = make([]any, 0)
-					for k := range item {
-						csvColumnKeys = append(csvColumnKeys, k)
+					csvColumnKeys = make([]string, 0)
+
+					for _, el := range els {
+						if el.Key() == "" {
+							continue
+						}
+
+						switch el.Value().Type {
+						case bsontype.Boolean,
+							bsontype.Decimal128,
+							bsontype.Double,
+							bsontype.Int32,
+							bsontype.Int64,
+							bsontype.Null,
+							bsontype.ObjectID,
+							bsontype.Regex,
+							bsontype.String,
+							bsontype.Symbol,
+							bsontype.Timestamp,
+							bsontype.Undefined:
+							csvColumnKeys = append(csvColumnKeys, el.Key())
+						}
+					}
+
+					runtime.LogDebugf(a.ctx, "Export csvColumnKeys: %v", csvColumnKeys)
+
+					if err := csvWriter.Write(csvColumnKeys); err != nil {
+						runtime.LogInfof(a.ctx, "Unable to write item %d to CSV while exporting: %s", index, err.Error())
+						zenity.Error(err.Error(), zenity.Title("Unable to write item %d to CSV"), zenity.ErrorIcon)
 					}
 				}
 
 				for _, k := range csvColumnKeys {
-					csvItem = append(csvItem, item[k].(string))
+					r, err := cur.Current.LookupErr(k)
+					if err != nil {
+						csvItem = append(csvItem, "")
+						continue
+					}
+
+					var v any
+					if err := r.Unmarshal(&v); err != nil {
+						zenity.Error(err.Error(), zenity.Title(fmt.Sprintf("Unable to unmarshal field %s", k)), zenity.ErrorIcon)
+						csvItem = append(csvItem, "")
+						continue
+					}
+
+					csvItem = append(csvItem, string(v.(string)))
 				}
 
 			default:
-				for _, v := range item {
-					csvItem = append(csvItem, v.(string))
-				}
+				// @todo
 			}
 
 			if err := csvWriter.Write(csvItem); err != nil {
-				runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to write item %d to CSV while exporting", index))
-				runtime.LogInfo(a.ctx, err.Error())
+				runtime.LogInfof(a.ctx, "Unable to write item %d to CSV while exporting: %s", index, err.Error())
 				zenity.Error(err.Error(), zenity.Title("Unable to write item %d to CSV"), zenity.ErrorIcon)
 			}
 
-		case ExportFormatJsonArray, ExportFormatJsonNewline:
-			itemJson, err := json.Marshal(item)
+			csvWriter.Flush()
+
+		case ExportFormatJsonArray, ExportFormatNdJson:
+			itemJson, err := bson.MarshalExtJSON(cur.Current, true, false)
 			if err != nil {
-				runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to marshal item %d to JSON while exporting", index))
-				runtime.LogInfo(a.ctx, err.Error())
+				runtime.LogInfof(a.ctx, "Unable to marshal item %d to JSON while exporting: %s", index, err.Error())
 				zenity.Error(err.Error(), zenity.Title("Unable to marshal item %d to JSON"), zenity.ErrorIcon)
 			}
 
-			file.Write(itemJson)
-			if settings.Format == ExportFormatJsonArray {
-				file.WriteString(",")
+			if (settings.Format == ExportFormatJsonArray) && (index != 0) {
+				file.WriteString(",\n")
 			}
-			file.WriteString("\n")
+
+			file.Write(itemJson)
+
+			if settings.Format == ExportFormatNdJson {
+				file.WriteString("\n")
+			}
+		}
+
+		index++
+
+		if count != 0 && pgr != nil {
+			p := (float32(index) / float32(count)) * 100.0
+			pgr.Value(int(p))
 		}
 	}
 
@@ -174,5 +290,12 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
 		file.WriteString("]\n")
 	}
 
+	if pgr != nil {
+		pgr.Complete()
+		pgr.Close()
+	}
+
+	a.ui.Reveal(settings.OutFile)
+	runtime.LogInfo(a.ctx, "Export succeeded")
 	return true
 }
diff --git a/internal/logger.go b/internal/logger.go
new file mode 100644
index 0000000..e069b42
--- /dev/null
+++ b/internal/logger.go
@@ -0,0 +1,80 @@
+package internal
+
+import (
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ncruces/zenity"
+)
+
+var showError = true
+
+type AppLogger struct {
+	directory string
+	filename  string
+	filepath  string
+}
+
+func NewAppLogger(directory, filename string) *AppLogger {
+	return &AppLogger{
+		directory: directory,
+		filename:  filename,
+		filepath:  path.Join(directory, filename),
+	}
+}
+
+func (l *AppLogger) Print(message string) {
+	os.MkdirAll(l.directory, os.ModePerm)
+	f, err := os.OpenFile(l.filepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+
+	if err != nil && showError {
+		zenity.Error(err.Error(), zenity.Title("Could not open logfile!"), zenity.ErrorIcon)
+		showError = false
+	}
+
+	if _, err = f.WriteString(message); err != nil {
+		if showError {
+			zenity.Error(err.Error(), zenity.Title("Could not write to logfile!"), zenity.ErrorIcon)
+			showError = false
+		} else {
+			showError = true
+		}
+	} else {
+		showError = true
+	}
+
+	f.Close()
+}
+
+func (l *AppLogger) Println(message string) {
+	l.Print(message + "\n")
+}
+
+func (l *AppLogger) Trace(message string) {
+	l.Println("TRACE | " + message)
+}
+
+func (l *AppLogger) Debug(message string) {
+	if strings.HasPrefix(message, "[ExternalAssetHandler]") {
+		return
+	}
+	l.Println("DEBUG | " + message)
+}
+
+func (l *AppLogger) Info(message string) {
+	l.Println("INFO  | " + message)
+}
+
+func (l *AppLogger) Warning(message string) {
+	l.Println("WARN  | " + message)
+}
+
+func (l *AppLogger) Error(message string) {
+	l.Println("ERROR | " + message)
+}
+
+func (l *AppLogger) Fatal(message string) {
+	l.Println("FATAL | " + message)
+	os.Exit(1)
+}
diff --git a/internal/ui/dialogs.go b/internal/ui/dialogs.go
index 0275340..1845abb 100644
--- a/internal/ui/dialogs.go
+++ b/internal/ui/dialogs.go
@@ -2,7 +2,7 @@ package ui
 
 import "github.com/ncruces/zenity"
 
-func (u *UI) OpenDirectory(id, title string) string {
+func (u *UI) OpenDirectory(title string) string {
 	if title == "" {
 		title = "Choose a directory"
 	}
diff --git a/main.go b/main.go
index b965331..e294f6f 100644
--- a/main.go
+++ b/main.go
@@ -3,8 +3,8 @@ package main
 import (
 	"context"
 	"embed"
-	"path"
 
+	"github.com/garraflavatra/rolens/internal"
 	"github.com/garraflavatra/rolens/internal/app"
 	uictrl "github.com/garraflavatra/rolens/internal/ui"
 	"github.com/ncruces/zenity"
@@ -54,7 +54,7 @@ func main() {
 		},
 		OnShutdown: app.Shutdown,
 
-		Logger:             logger.NewFileLogger(path.Join(app.Env.LogDirectory, "rolens.log")),
+		Logger:             internal.NewAppLogger(app.Env.LogDirectory, "rolens.log"),
 		LogLevel:           logger.TRACE,
 		LogLevelProduction: logger.INFO,