romein/rolens
1
0
Fork 0
mirror of https://github.com/garraflavatra/rolens.git synced 2025-01-18 13:07:58 +00:00
Code Issues Releases Activity

Improvememts to export tool and logging

Browse Source
This commit is contained in:
Romein van Buren 2023-05-28 21:54:40 +02:00
parent a5eadbf456
commit 82d880dbff
Signed by: romein
GPG Key ID: 0EFF8478ADDF6C49
9 changed files with 283 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
frontend/src/organisms/connection/collection/components/export.svelte
View File

@ -1,8 +1,8 @@
<script>
import Icon from '$components/icon.svelte';
import Modal from '$components/modal.svelte';
import { startProgress } from '$lib/progress';
import views from '$lib/stores/views';
import { PerformFindExport } from '$wails/go/app/App';
import { createEventDispatcher } from 'svelte';
export let info;
@ -11,16 +11,21 @@
const dispatch = createEventDispatcher();
let viewKey = collection.viewKey;
$: viewKey = collection.viewKey;
$: if (info) {
info.viewKey = viewKey;
}
async function performExport() {
const progress = startProgress('Performing export…');
info.view = $views[viewKey];
//...
progress.end();
const success = await PerformFindExport(collection.hostKey, collection.dbKey, collection.key, JSON.stringify(info));
if (success) {
info = undefined;
}
}
</script>
<Modal bind:show={info} title="Export results" width="400px">
<Modal bind:show={info} title="Export results" width="450px">
<form on:submit|preventDefault={performExport}>
<label class="field">
<span class="label">Export</span>
@ -30,14 +35,16 @@
<option value="querylimitskip">all records matching query, considering limit and skip</option>
</select>
</label>
<label class="field">
<span class="label">Format</span>
<select bind:value={info.format}>
<option value="jsonarray">JSON array</option>
<option value="jsonnewline">JSON: newline-separated objects</option>
<option value="ndjson">Newline delimited JSON</option>
<option value="csv">CSV</option>
</select>
</label>
<label class="field">
<span class="label">View to use</span>
<select bind:value={viewKey}>
@ -49,6 +56,11 @@
<Icon name="cog" />
</button>
</label>
<button class="btn" type="submit">
<Icon name="play" />
Start export
</button>
</form>
</Modal>

2
frontend/wailsjs/go/models.ts generated
View File

@ -9,6 +9,7 @@ export namespace app {
homeDirectory: string;
dataDirectory: string;
logDirectory: string;
downloadDirectory: string;
static createFrom(source: any = {}) {
return new EnvironmentInfo(source);
@ -24,6 +25,7 @@ export namespace app {
this.homeDirectory = source["homeDirectory"];
this.dataDirectory = source["dataDirectory"];
this.logDirectory = source["logDirectory"];
this.downloadDirectory = source["downloadDirectory"];
}
}
export class QueryResult {

2
frontend/wailsjs/go/ui/UI.d.ts generated vendored
View File

@ -6,7 +6,7 @@ export function Beep():Promise<void>;
export function EnterText(arg1:string,arg2:string,arg3:string):Promise<string>;
export function OpenDirectory(arg1:string,arg2:string):Promise<string>;
export function OpenDirectory(arg1:string):Promise<string>;
export function Reveal(arg1:string):Promise<void>;

4
frontend/wailsjs/go/ui/UI.js generated
View File

@ -10,8 +10,8 @@ export function EnterText(arg1, arg2, arg3) {
return window['go']['ui']['UI']['EnterText'](arg1, arg2, arg3);
}
export function OpenDirectory(arg1, arg2) {
return window['go']['ui']['UI']['OpenDirectory'](arg1, arg2);
export function OpenDirectory(arg1) {
return window['go']['ui']['UI']['OpenDirectory'](arg1);
}
export function Reveal(arg1) {

10
internal/app/app.go
View File

@ -22,9 +22,10 @@ type EnvironmentInfo struct {
HasMongoExport bool `json:"hasMongoExport"`
HasMongoDump bool `json:"hasMongoDump"`
HomeDirectory string `json:"homeDirectory"`
DataDirectory string `json:"dataDirectory"`
LogDirectory string `json:"logDirectory"`
HomeDirectory string `json:"homeDirectory"`
DataDirectory string `json:"dataDirectory"`
LogDirectory string `json:"logDirectory"`
DownloadDirectory string `json:"downloadDirectory"`
}
type App struct {
@ -54,12 +55,15 @@ func NewApp() *App {
case "windows":
a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/AppData/Local/Rolens")
a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/AppData/Local/Rolens/Logs")
a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
case "darwin":
a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/Library/Application Support/Rolens")
a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/Library/Logs/Rolens")
a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
case "linux":
a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/.config/rolens")
a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/.config/rolens/logs")
a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
default:
panic(errors.New("unsupported platform"))
}

217
internal/app/collection_find_export.go
View File

@ -9,6 +9,7 @@ import (
"github.com/ncruces/zenity"
"github.com/wailsapp/wails/v2/pkg/runtime"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsontype"
"go.mongodb.org/mongo-driver/mongo/options"
)
@ -20,9 +21,9 @@ const (
ExportContentsQuery ExportContents = "query"
ExportContentsQueryLimitSkip ExportContents = "querylimitskip"
ExportFormatJsonArray ExportFormat = "jsonarray"
ExportFormatJsonNewline ExportFormat = "jsonnewline"
ExportFormatCsv ExportFormat = "csv"
ExportFormatJsonArray ExportFormat = "jsonarray"
ExportFormatNdJson ExportFormat = "ndjson"
ExportFormatCsv ExportFormat = "csv"
)
type ExportSettings struct {
@ -30,41 +31,101 @@ type ExportSettings struct {
Format ExportFormat `json:"format"`
ViewKey string `json:"viewKey"`
QueryJson string `json:"query"`
Limit int64 `json:"limit"`
Skip int64 `json:"skip"`
Limit uint `json:"limit"`
Skip uint `json:"skip"`
OutFile string `json:"outfile"`
}
func getptr[T any](v T) *T {
return &v
}
func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bool {
runtime.LogInfof(a.ctx, "Export started for %s/%s/%s. Settings: %s", hostKey, dbKey, collKey, settingsJson)
var settings ExportSettings
if err := json.Unmarshal([]byte(settingsJson), &settings); err != nil {
runtime.LogWarning(a.ctx, "Could not parse export settings:")
runtime.LogWarning(a.ctx, err.Error())
runtime.LogWarningf(a.ctx, "Could not parse export settings: %s", err.Error())
zenity.Error(err.Error(), zenity.Title("Couldn't parse export settings!"), zenity.ErrorIcon)
return false
}
if _, err := os.Stat(settings.OutFile); err == nil {
zenity.Error(fmt.Sprintf("File %s already exists, export aborted.", settings.OutFile), zenity.ErrorIcon)
return false
switch settings.Contents {
case ExportContentsAll:
settings.QueryJson = "{}"
settings.Limit = 0
settings.Skip = 0
case ExportContentsQuery:
settings.Limit = 0
settings.Skip = 0
case ExportContentsQueryLimitSkip:
}
views, err := a.Views()
if err != nil {
runtime.LogWarningf(a.ctx, "Export: error while retrieving view: %s", err.Error())
return false
}
view, found := views[settings.ViewKey]
if !found {
zenity.Error(fmt.Sprintf("View %s is not known", settings.ViewKey), zenity.ErrorIcon)
runtime.LogDebugf(a.ctx, "Export: unknown view %s", settings.ViewKey)
return false
}
var fileFilter runtime.FileFilter
defaultFilename := ""
switch settings.Format {
case ExportFormatCsv:
defaultFilename = "export.csv"
fileFilter = runtime.FileFilter{
DisplayName: "CSV files (*.csv)",
Pattern: "*.csv",
}
case ExportFormatJsonArray:
defaultFilename = "export.json"
fileFilter = runtime.FileFilter{
DisplayName: "JSON files (*.json)",
Pattern: "*.json",
}
case ExportFormatNdJson:
defaultFilename = "export.ndjson"
fileFilter = runtime.FileFilter{
DisplayName: "Newline delimited JSON files (*.ndjson)",
Pattern: "*.ndjson",
}
}
settings.OutFile, err = runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
Title: "Choose export destination",
DefaultDirectory: a.Env.DownloadDirectory,
CanCreateDirectories: true,
DefaultFilename: defaultFilename,
Filters: []runtime.FileFilter{fileFilter},
})
if err != nil {
zenity.Error("An error occured while choosing the export destination", zenity.ErrorIcon)
runtime.LogWarningf(a.ctx, "Export: error while choosing export destination: %s", err.Error())
return false
}
if settings.OutFile == "" {
zenity.Error("You must specify an export destination.", zenity.ErrorIcon)
runtime.LogDebug(a.ctx, "Export: no destination specified")
return false
}
if _, err := os.Stat(settings.OutFile); err == nil {
zenity.Error(fmt.Sprintf("File %s already exists, export aborted.", settings.OutFile), zenity.ErrorIcon)
runtime.LogDebugf(a.ctx, "Export: destination %s already exists", settings.OutFile)
return false
}
var query bson.M
if settings.Contents != ExportContentsAll {
if err = bson.UnmarshalExtJSON([]byte(settings.QueryJson), true, &query); err != nil {
runtime.LogInfo(a.ctx, "Invalid find query (exporting):")
runtime.LogInfo(a.ctx, err.Error())
runtime.LogDebugf(a.ctx, "Invalid find query (exporting): %s", settings.QueryJson)
zenity.Error(err.Error(), zenity.Title("Invalid query"), zenity.ErrorIcon)
return false
}
@ -76,22 +137,32 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
}
defer close()
projection := bson.M{}
pgr, _ := zenity.Progress(zenity.Title("Performing export…"))
projection := bson.M{}
if settings.ViewKey != "list" {
for _, col := range view.Columns {
projection[col.Key] = ""
}
}
var count uint = 0
if settings.Limit != 0 {
count = uint(settings.Limit)
} else {
c, _ := client.Database(dbKey).Collection(collKey).CountDocuments(ctx, query, &options.CountOptions{
Skip: getptr(int64(settings.Skip)),
})
count = uint(c)
}
cur, err := client.Database(dbKey).Collection(collKey).Find(ctx, query, &options.FindOptions{
Skip: &settings.Skip,
Limit: &settings.Limit,
Skip: getptr(int64(settings.Skip)),
Limit: getptr(int64(settings.Limit)),
Projection: projection,
})
if err != nil {
runtime.LogInfo(a.ctx, "Unable to get cursor while exporting:")
runtime.LogInfo(a.ctx, err.Error())
runtime.LogInfof(a.ctx, "Export: unable to get cursor while exporting: %s", err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to get cursor"), zenity.ErrorIcon)
return false
}
@ -99,74 +170,119 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
file, err := os.OpenFile(settings.OutFile, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
zenity.Error(fmt.Sprintf(err.Error(), zenity.Title("Error while opening file"), settings.OutFile), zenity.ErrorIcon)
runtime.LogDebugf(a.ctx, "Export: unable to open file %s", settings.OutFile)
return false
}
defer file.Close()
var csvWriter *csv.Writer
var csvColumnKeys []any
var csvColumnKeys []string
var index uint = 0
switch settings.Format {
case ExportFormatJsonArray:
file.WriteString("[\n")
file.WriteString("[")
case ExportFormatCsv:
csvWriter = csv.NewWriter(file)
}
index := -1
for cur.Next(ctx) {
index++
var item map[any]interface{}
if err := bson.Unmarshal(cur.Current, &item); err != nil {
runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to unmarshal item %d while exporting", index))
runtime.LogInfo(a.ctx, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to unmarshal item %d"), zenity.ErrorIcon)
continue
}
switch settings.Format {
case ExportFormatCsv:
els, err := cur.Current.Elements()
if err != nil {
zenity.Error(err.Error(), zenity.Title("BSON invalid"), zenity.ErrorIcon)
}
csvItem := make([]string, 0)
switch settings.ViewKey {
case "list":
if csvColumnKeys == nil {
csvColumnKeys = make([]any, 0)
for k := range item {
csvColumnKeys = append(csvColumnKeys, k)
csvColumnKeys = make([]string, 0)
for _, el := range els {
if el.Key() == "" {
continue
}
switch el.Value().Type {
case bsontype.Boolean,
bsontype.Decimal128,
bsontype.Double,
bsontype.Int32,
bsontype.Int64,
bsontype.Null,
bsontype.ObjectID,
bsontype.Regex,
bsontype.String,
bsontype.Symbol,
bsontype.Timestamp,
bsontype.Undefined:
csvColumnKeys = append(csvColumnKeys, el.Key())
}
}
runtime.LogDebugf(a.ctx, "Export csvColumnKeys: %v", csvColumnKeys)
if err := csvWriter.Write(csvColumnKeys); err != nil {
runtime.LogInfof(a.ctx, "Unable to write item %d to CSV while exporting: %s", index, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to write item %d to CSV"), zenity.ErrorIcon)
}
}
for _, k := range csvColumnKeys {
csvItem = append(csvItem, item[k].(string))
r, err := cur.Current.LookupErr(k)
if err != nil {
csvItem = append(csvItem, "")
continue
}
var v any
if err := r.Unmarshal(&v); err != nil {
zenity.Error(err.Error(), zenity.Title(fmt.Sprintf("Unable to unmarshal field %s", k)), zenity.ErrorIcon)
csvItem = append(csvItem, "")
continue
}
csvItem = append(csvItem, string(v.(string)))
}
default:
for _, v := range item {
csvItem = append(csvItem, v.(string))
}
// @todo
}
if err := csvWriter.Write(csvItem); err != nil {
runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to write item %d to CSV while exporting", index))
runtime.LogInfo(a.ctx, err.Error())
runtime.LogInfof(a.ctx, "Unable to write item %d to CSV while exporting: %s", index, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to write item %d to CSV"), zenity.ErrorIcon)
}
case ExportFormatJsonArray, ExportFormatJsonNewline:
itemJson, err := json.Marshal(item)
csvWriter.Flush()
case ExportFormatJsonArray, ExportFormatNdJson:
itemJson, err := bson.MarshalExtJSON(cur.Current, true, false)
if err != nil {
runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to marshal item %d to JSON while exporting", index))
runtime.LogInfo(a.ctx, err.Error())
runtime.LogInfof(a.ctx, "Unable to marshal item %d to JSON while exporting: %s", index, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to marshal item %d to JSON"), zenity.ErrorIcon)
}
file.Write(itemJson)
if settings.Format == ExportFormatJsonArray {
file.WriteString(",")
if (settings.Format == ExportFormatJsonArray) && (index != 0) {
file.WriteString(",\n")
}
file.WriteString("\n")
file.Write(itemJson)
if settings.Format == ExportFormatNdJson {
file.WriteString("\n")
}
}
index++
if count != 0 && pgr != nil {
p := (float32(index) / float32(count)) * 100.0
pgr.Value(int(p))
}
}
@ -174,5 +290,12 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
file.WriteString("]\n")
}
if pgr != nil {
pgr.Complete()
pgr.Close()
}
a.ui.Reveal(settings.OutFile)
runtime.LogInfo(a.ctx, "Export succeeded")
return true
}

80
internal/logger.go Normal file
View File

@ -0,0 +1,80 @@
package internal
import (
"os"
"path"
"strings"
"github.com/ncruces/zenity"
)
var showError = true
type AppLogger struct {
directory string
filename string
filepath string
}
func NewAppLogger(directory, filename string) *AppLogger {
return &AppLogger{
directory: directory,
filename: filename,
filepath: path.Join(directory, filename),
}
}
func (l *AppLogger) Print(message string) {
os.MkdirAll(l.directory, os.ModePerm)
f, err := os.OpenFile(l.filepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil && showError {
zenity.Error(err.Error(), zenity.Title("Could not open logfile!"), zenity.ErrorIcon)
showError = false
}
if _, err = f.WriteString(message); err != nil {
if showError {
zenity.Error(err.Error(), zenity.Title("Could not write to logfile!"), zenity.ErrorIcon)
showError = false
} else {
showError = true
}
} else {
showError = true
}
f.Close()
}
func (l *AppLogger) Println(message string) {
l.Print(message + "\n")
}
func (l *AppLogger) Trace(message string) {
l.Println("TRACE | " + message)
}
func (l *AppLogger) Debug(message string) {
if strings.HasPrefix(message, "[ExternalAssetHandler]") {
return
}
l.Println("DEBUG | " + message)
}
func (l *AppLogger) Info(message string) {
l.Println("INFO | " + message)
}
func (l *AppLogger) Warning(message string) {
l.Println("WARN | " + message)
}
func (l *AppLogger) Error(message string) {
l.Println("ERROR | " + message)
}
func (l *AppLogger) Fatal(message string) {
l.Println("FATAL | " + message)
os.Exit(1)
}

2
internal/ui/dialogs.go
View File

@ -2,7 +2,7 @@ package ui
import "github.com/ncruces/zenity"
func (u *UI) OpenDirectory(id, title string) string {
func (u *UI) OpenDirectory(title string) string {
if title == "" {
title = "Choose a directory"
}

4
main.go
View File

@ -3,8 +3,8 @@ package main
import (
"context"
"embed"
"path"
"github.com/garraflavatra/rolens/internal"
"github.com/garraflavatra/rolens/internal/app"
uictrl "github.com/garraflavatra/rolens/internal/ui"
"github.com/ncruces/zenity"
@ -54,7 +54,7 @@ func main() {
},
OnShutdown: app.Shutdown,
Logger: logger.NewFileLogger(path.Join(app.Env.LogDirectory, "rolens.log")),
Logger: internal.NewAppLogger(app.Env.LogDirectory, "rolens.log"),
LogLevel: logger.TRACE,
LogLevelProduction: logger.INFO,