romein/rolens
1
0
Fork 0
mirror of https://github.com/garraflavatra/rolens.git synced 2025-06-28 05:25:11 +00:00
Code Issues Releases Activity

Improvememts to export tool and logging

Browse Source
This commit is contained in:
Romein van Buren
2023-05-28 21:54:40 +02:00
parent a5eadbf456
commit 82d880dbff
9 changed files with 283 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/app/app.go
View File

@ -22,9 +22,10 @@ type EnvironmentInfo struct {
HasMongoExport bool `json:"hasMongoExport"`
HasMongoDump bool `json:"hasMongoDump"`
HomeDirectory string `json:"homeDirectory"`
DataDirectory string `json:"dataDirectory"`
LogDirectory string `json:"logDirectory"`
HomeDirectory string `json:"homeDirectory"`
DataDirectory string `json:"dataDirectory"`
LogDirectory string `json:"logDirectory"`
DownloadDirectory string `json:"downloadDirectory"`
}
type App struct {
@ -54,12 +55,15 @@ func NewApp() *App {
case "windows":
a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/AppData/Local/Rolens")
a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/AppData/Local/Rolens/Logs")
a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
case "darwin":
a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/Library/Application Support/Rolens")
a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/Library/Logs/Rolens")
a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
case "linux":
a.Env.DataDirectory = filepath.Join(a.Env.HomeDirectory, "/.config/rolens")
a.Env.LogDirectory = filepath.Join(a.Env.HomeDirectory, "/.config/rolens/logs")
a.Env.DownloadDirectory = filepath.Join(a.Env.HomeDirectory, "/Downloads")
default:
panic(errors.New("unsupported platform"))
}

217
internal/app/collection_find_export.go
View File

@ -9,6 +9,7 @@ import (
"github.com/ncruces/zenity"
"github.com/wailsapp/wails/v2/pkg/runtime"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsontype"
"go.mongodb.org/mongo-driver/mongo/options"
)
@ -20,9 +21,9 @@ const (
ExportContentsQuery ExportContents = "query"
ExportContentsQueryLimitSkip ExportContents = "querylimitskip"
ExportFormatJsonArray ExportFormat = "jsonarray"
ExportFormatJsonNewline ExportFormat = "jsonnewline"
ExportFormatCsv ExportFormat = "csv"
ExportFormatJsonArray ExportFormat = "jsonarray"
ExportFormatNdJson ExportFormat = "ndjson"
ExportFormatCsv ExportFormat = "csv"
)
type ExportSettings struct {
@ -30,41 +31,101 @@ type ExportSettings struct {
Format ExportFormat `json:"format"`
ViewKey string `json:"viewKey"`
QueryJson string `json:"query"`
Limit int64 `json:"limit"`
Skip int64 `json:"skip"`
Limit uint `json:"limit"`
Skip uint `json:"skip"`
OutFile string `json:"outfile"`
}
func getptr[T any](v T) *T {
return &v
}
func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bool {
runtime.LogInfof(a.ctx, "Export started for %s/%s/%s. Settings: %s", hostKey, dbKey, collKey, settingsJson)
var settings ExportSettings
if err := json.Unmarshal([]byte(settingsJson), &settings); err != nil {
runtime.LogWarning(a.ctx, "Could not parse export settings:")
runtime.LogWarning(a.ctx, err.Error())
runtime.LogWarningf(a.ctx, "Could not parse export settings: %s", err.Error())
zenity.Error(err.Error(), zenity.Title("Couldn't parse export settings!"), zenity.ErrorIcon)
return false
}
if _, err := os.Stat(settings.OutFile); err == nil {
zenity.Error(fmt.Sprintf("File %s already exists, export aborted.", settings.OutFile), zenity.ErrorIcon)
return false
switch settings.Contents {
case ExportContentsAll:
settings.QueryJson = "{}"
settings.Limit = 0
settings.Skip = 0
case ExportContentsQuery:
settings.Limit = 0
settings.Skip = 0
case ExportContentsQueryLimitSkip:
}
views, err := a.Views()
if err != nil {
runtime.LogWarningf(a.ctx, "Export: error while retrieving view: %s", err.Error())
return false
}
view, found := views[settings.ViewKey]
if !found {
zenity.Error(fmt.Sprintf("View %s is not known", settings.ViewKey), zenity.ErrorIcon)
runtime.LogDebugf(a.ctx, "Export: unknown view %s", settings.ViewKey)
return false
}
var fileFilter runtime.FileFilter
defaultFilename := ""
switch settings.Format {
case ExportFormatCsv:
defaultFilename = "export.csv"
fileFilter = runtime.FileFilter{
DisplayName: "CSV files (*.csv)",
Pattern: "*.csv",
}
case ExportFormatJsonArray:
defaultFilename = "export.json"
fileFilter = runtime.FileFilter{
DisplayName: "JSON files (*.json)",
Pattern: "*.json",
}
case ExportFormatNdJson:
defaultFilename = "export.ndjson"
fileFilter = runtime.FileFilter{
DisplayName: "Newline delimited JSON files (*.ndjson)",
Pattern: "*.ndjson",
}
}
settings.OutFile, err = runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
Title: "Choose export destination",
DefaultDirectory: a.Env.DownloadDirectory,
CanCreateDirectories: true,
DefaultFilename: defaultFilename,
Filters: []runtime.FileFilter{fileFilter},
})
if err != nil {
zenity.Error("An error occured while choosing the export destination", zenity.ErrorIcon)
runtime.LogWarningf(a.ctx, "Export: error while choosing export destination: %s", err.Error())
return false
}
if settings.OutFile == "" {
zenity.Error("You must specify an export destination.", zenity.ErrorIcon)
runtime.LogDebug(a.ctx, "Export: no destination specified")
return false
}
if _, err := os.Stat(settings.OutFile); err == nil {
zenity.Error(fmt.Sprintf("File %s already exists, export aborted.", settings.OutFile), zenity.ErrorIcon)
runtime.LogDebugf(a.ctx, "Export: destination %s already exists", settings.OutFile)
return false
}
var query bson.M
if settings.Contents != ExportContentsAll {
if err = bson.UnmarshalExtJSON([]byte(settings.QueryJson), true, &query); err != nil {
runtime.LogInfo(a.ctx, "Invalid find query (exporting):")
runtime.LogInfo(a.ctx, err.Error())
runtime.LogDebugf(a.ctx, "Invalid find query (exporting): %s", settings.QueryJson)
zenity.Error(err.Error(), zenity.Title("Invalid query"), zenity.ErrorIcon)
return false
}
@ -76,22 +137,32 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
}
defer close()
projection := bson.M{}
pgr, _ := zenity.Progress(zenity.Title("Performing export…"))
projection := bson.M{}
if settings.ViewKey != "list" {
for _, col := range view.Columns {
projection[col.Key] = ""
}
}
var count uint = 0
if settings.Limit != 0 {
count = uint(settings.Limit)
} else {
c, _ := client.Database(dbKey).Collection(collKey).CountDocuments(ctx, query, &options.CountOptions{
Skip: getptr(int64(settings.Skip)),
})
count = uint(c)
}
cur, err := client.Database(dbKey).Collection(collKey).Find(ctx, query, &options.FindOptions{
Skip: &settings.Skip,
Limit: &settings.Limit,
Skip: getptr(int64(settings.Skip)),
Limit: getptr(int64(settings.Limit)),
Projection: projection,
})
if err != nil {
runtime.LogInfo(a.ctx, "Unable to get cursor while exporting:")
runtime.LogInfo(a.ctx, err.Error())
runtime.LogInfof(a.ctx, "Export: unable to get cursor while exporting: %s", err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to get cursor"), zenity.ErrorIcon)
return false
}
@ -99,74 +170,119 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
file, err := os.OpenFile(settings.OutFile, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
zenity.Error(fmt.Sprintf(err.Error(), zenity.Title("Error while opening file"), settings.OutFile), zenity.ErrorIcon)
runtime.LogDebugf(a.ctx, "Export: unable to open file %s", settings.OutFile)
return false
}
defer file.Close()
var csvWriter *csv.Writer
var csvColumnKeys []any
var csvColumnKeys []string
var index uint = 0
switch settings.Format {
case ExportFormatJsonArray:
file.WriteString("[\n")
file.WriteString("[")
case ExportFormatCsv:
csvWriter = csv.NewWriter(file)
}
index := -1
for cur.Next(ctx) {
index++
var item map[any]interface{}
if err := bson.Unmarshal(cur.Current, &item); err != nil {
runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to unmarshal item %d while exporting", index))
runtime.LogInfo(a.ctx, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to unmarshal item %d"), zenity.ErrorIcon)
continue
}
switch settings.Format {
case ExportFormatCsv:
els, err := cur.Current.Elements()
if err != nil {
zenity.Error(err.Error(), zenity.Title("BSON invalid"), zenity.ErrorIcon)
}
csvItem := make([]string, 0)
switch settings.ViewKey {
case "list":
if csvColumnKeys == nil {
csvColumnKeys = make([]any, 0)
for k := range item {
csvColumnKeys = append(csvColumnKeys, k)
csvColumnKeys = make([]string, 0)
for _, el := range els {
if el.Key() == "" {
continue
}
switch el.Value().Type {
case bsontype.Boolean,
bsontype.Decimal128,
bsontype.Double,
bsontype.Int32,
bsontype.Int64,
bsontype.Null,
bsontype.ObjectID,
bsontype.Regex,
bsontype.String,
bsontype.Symbol,
bsontype.Timestamp,
bsontype.Undefined:
csvColumnKeys = append(csvColumnKeys, el.Key())
}
}
runtime.LogDebugf(a.ctx, "Export csvColumnKeys: %v", csvColumnKeys)
if err := csvWriter.Write(csvColumnKeys); err != nil {
runtime.LogInfof(a.ctx, "Unable to write item %d to CSV while exporting: %s", index, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to write item %d to CSV"), zenity.ErrorIcon)
}
}
for _, k := range csvColumnKeys {
csvItem = append(csvItem, item[k].(string))
r, err := cur.Current.LookupErr(k)
if err != nil {
csvItem = append(csvItem, "")
continue
}
var v any
if err := r.Unmarshal(&v); err != nil {
zenity.Error(err.Error(), zenity.Title(fmt.Sprintf("Unable to unmarshal field %s", k)), zenity.ErrorIcon)
csvItem = append(csvItem, "")
continue
}
csvItem = append(csvItem, string(v.(string)))
}
default:
for _, v := range item {
csvItem = append(csvItem, v.(string))
}
// @todo
}
if err := csvWriter.Write(csvItem); err != nil {
runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to write item %d to CSV while exporting", index))
runtime.LogInfo(a.ctx, err.Error())
runtime.LogInfof(a.ctx, "Unable to write item %d to CSV while exporting: %s", index, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to write item %d to CSV"), zenity.ErrorIcon)
}
case ExportFormatJsonArray, ExportFormatJsonNewline:
itemJson, err := json.Marshal(item)
csvWriter.Flush()
case ExportFormatJsonArray, ExportFormatNdJson:
itemJson, err := bson.MarshalExtJSON(cur.Current, true, false)
if err != nil {
runtime.LogInfo(a.ctx, fmt.Sprintf("Unable to marshal item %d to JSON while exporting", index))
runtime.LogInfo(a.ctx, err.Error())
runtime.LogInfof(a.ctx, "Unable to marshal item %d to JSON while exporting: %s", index, err.Error())
zenity.Error(err.Error(), zenity.Title("Unable to marshal item %d to JSON"), zenity.ErrorIcon)
}
file.Write(itemJson)
if settings.Format == ExportFormatJsonArray {
file.WriteString(",")
if (settings.Format == ExportFormatJsonArray) && (index != 0) {
file.WriteString(",\n")
}
file.WriteString("\n")
file.Write(itemJson)
if settings.Format == ExportFormatNdJson {
file.WriteString("\n")
}
}
index++
if count != 0 && pgr != nil {
p := (float32(index) / float32(count)) * 100.0
pgr.Value(int(p))
}
}
@ -174,5 +290,12 @@ func (a *App) PerformFindExport(hostKey, dbKey, collKey, settingsJson string) bo
file.WriteString("]\n")
}
if pgr != nil {
pgr.Complete()
pgr.Close()
}
a.ui.Reveal(settings.OutFile)
runtime.LogInfo(a.ctx, "Export succeeded")
return true
}

80
internal/logger.go Normal file
View File

@ -0,0 +1,80 @@
package internal
import (
"os"
"path"
"strings"
"github.com/ncruces/zenity"
)
var showError = true
type AppLogger struct {
directory string
filename string
filepath string
}
func NewAppLogger(directory, filename string) *AppLogger {
return &AppLogger{
directory: directory,
filename: filename,
filepath: path.Join(directory, filename),
}
}
func (l *AppLogger) Print(message string) {
os.MkdirAll(l.directory, os.ModePerm)
f, err := os.OpenFile(l.filepath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil && showError {
zenity.Error(err.Error(), zenity.Title("Could not open logfile!"), zenity.ErrorIcon)
showError = false
}
if _, err = f.WriteString(message); err != nil {
if showError {
zenity.Error(err.Error(), zenity.Title("Could not write to logfile!"), zenity.ErrorIcon)
showError = false
} else {
showError = true
}
} else {
showError = true
}
f.Close()
}
func (l *AppLogger) Println(message string) {
l.Print(message + "\n")
}
func (l *AppLogger) Trace(message string) {
l.Println("TRACE | " + message)
}
func (l *AppLogger) Debug(message string) {
if strings.HasPrefix(message, "[ExternalAssetHandler]") {
return
}
l.Println("DEBUG | " + message)
}
func (l *AppLogger) Info(message string) {
l.Println("INFO | " + message)
}
func (l *AppLogger) Warning(message string) {
l.Println("WARN | " + message)
}
func (l *AppLogger) Error(message string) {
l.Println("ERROR | " + message)
}
func (l *AppLogger) Fatal(message string) {
l.Println("FATAL | " + message)
os.Exit(1)
}

2
internal/ui/dialogs.go
View File

@ -2,7 +2,7 @@ package ui
import "github.com/ncruces/zenity"
func (u *UI) OpenDirectory(id, title string) string {
func (u *UI) OpenDirectory(title string) string {
if title == "" {
title = "Choose a directory"
}