|
@@ -11,36 +11,43 @@ import (
|
|
"encoding/csv"
|
|
"encoding/csv"
|
|
"fmt"
|
|
"fmt"
|
|
"io"
|
|
"io"
|
|
- "log"
|
|
|
|
"os"
|
|
"os"
|
|
"os/signal"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"path/filepath"
|
|
"regexp"
|
|
"regexp"
|
|
"strings"
|
|
"strings"
|
|
|
|
+ "time"
|
|
|
|
|
|
_ "github.com/mattn/go-sqlite3"
|
|
_ "github.com/mattn/go-sqlite3"
|
|
|
|
+ logger "idio.link/go/logger/v3"
|
|
|
|
|
|
"golang.org/x/term"
|
|
"golang.org/x/term"
|
|
)
|
|
)
|
|
|
|
|
|
var (
|
|
var (
|
|
- Version string
|
|
|
|
- Build string
|
|
|
|
|
|
+ Version, Build string
|
|
)
|
|
)
|
|
|
|
|
|
const MaxRecords = 1_000_000_000
|
|
const MaxRecords = 1_000_000_000
|
|
|
|
|
|
func main() {
|
|
func main() {
|
|
|
|
+ log := logger.NewLogger()
|
|
|
|
+
|
|
if len(os.Args) != 2 {
|
|
if len(os.Args) != 2 {
|
|
- log.Printf("%s %s", Version, Build)
|
|
|
|
- log.Printf("usage: %s <csv_path>", os.Args[0])
|
|
|
|
|
|
+ if Build == "" {
|
|
|
|
+ Version = "dev"
|
|
|
|
+ Build = time.Now().UTC().Format(time.RFC3339)
|
|
|
|
+ }
|
|
|
|
+ fmt.Printf("%s %s\n", Version, Build)
|
|
|
|
+ fmt.Printf("usage: %s <csv_path>\n", os.Args[0])
|
|
os.Exit(1)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
csvPath, err := filepath.Abs(os.Args[1])
|
|
csvPath, err := filepath.Abs(os.Args[1])
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("resolve absolute path for '%s': %v", os.Args[1], err))
|
|
|
|
|
|
+ log.Fatal("resolve absolute path for '%s': %v", os.Args[1], err)
|
|
}
|
|
}
|
|
- ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
|
|
+ ctx := context.WithValue(context.Background(), "log", log)
|
|
|
|
+ ctx, cancel := context.WithCancel(ctx)
|
|
|
|
|
|
go func() {
|
|
go func() {
|
|
_, stop := signal.NotifyContext(
|
|
_, stop := signal.NotifyContext(
|
|
@@ -51,18 +58,18 @@ func main() {
|
|
sig := make(chan os.Signal, 1)
|
|
sig := make(chan os.Signal, 1)
|
|
<-sig
|
|
<-sig
|
|
stop()
|
|
stop()
|
|
- log.Print("caught signal; shutting down")
|
|
|
|
|
|
+ log.Info("caught signal; shutting down")
|
|
cancel()
|
|
cancel()
|
|
}()
|
|
}()
|
|
|
|
|
|
// open csv for read
|
|
// open csv for read
|
|
f, err := os.Open(os.Args[1])
|
|
f, err := os.Open(os.Args[1])
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("open csv: %v", err))
|
|
|
|
|
|
+ log.Fatal("open csv: %v", err)
|
|
}
|
|
}
|
|
defer func() {
|
|
defer func() {
|
|
if err := f.Close(); err != nil {
|
|
if err := f.Close(); err != nil {
|
|
- log.Printf("close csv '%s': %v", csvPath, err)
|
|
|
|
|
|
+ log.Error("close csv '%s': %v", csvPath, err)
|
|
}
|
|
}
|
|
}()
|
|
}()
|
|
r := csv.NewReader(f)
|
|
r := csv.NewReader(f)
|
|
@@ -76,11 +83,11 @@ func main() {
|
|
params := "?_synchronous=0&_journal_mode=OFF&_temp_store=2"
|
|
params := "?_synchronous=0&_journal_mode=OFF&_temp_store=2"
|
|
db, err := sql.Open("sqlite3", dbPath+params)
|
|
db, err := sql.Open("sqlite3", dbPath+params)
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("open db '%s': %v", dbPath, err))
|
|
|
|
|
|
+ log.Fatal("open db '%s': %v", dbPath, err)
|
|
}
|
|
}
|
|
defer func() {
|
|
defer func() {
|
|
if err := db.Close(); err != nil {
|
|
if err := db.Close(); err != nil {
|
|
- log.Printf("close db '%s': %v", dbPath, err)
|
|
|
|
|
|
+ log.Error("close db '%s': %v", dbPath, err)
|
|
}
|
|
}
|
|
}()
|
|
}()
|
|
|
|
|
|
@@ -90,7 +97,7 @@ func main() {
|
|
|
|
|
|
rec, err := r.Read()
|
|
rec, err := r.Read()
|
|
if err != nil && err != io.EOF {
|
|
if err != nil && err != io.EOF {
|
|
- log.Fatal(fmt.Sprintf("read csv '%s': %v", csvPath, err))
|
|
|
|
|
|
+ log.Fatal("read csv '%s': %v", csvPath, err)
|
|
}
|
|
}
|
|
for _, f := range rec {
|
|
for _, f := range rec {
|
|
f = scrubName(f)
|
|
f = scrubName(f)
|
|
@@ -98,22 +105,22 @@ func main() {
|
|
}
|
|
}
|
|
_, err = createTable(ctx, db, name, headers)
|
|
_, err = createTable(ctx, db, name, headers)
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("create table '%s': %v", name, err))
|
|
|
|
|
|
+ log.Fatal("create table '%s': %v", name, err)
|
|
}
|
|
}
|
|
insert, err = genInsert(ctx, db, name, headers)
|
|
insert, err = genInsert(ctx, db, name, headers)
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("prepare insert: %v", err))
|
|
|
|
|
|
+ log.Fatal("prepare insert: %v", err)
|
|
}
|
|
}
|
|
defer func() {
|
|
defer func() {
|
|
if err := insert.Close(); err != nil {
|
|
if err := insert.Close(); err != nil {
|
|
- log.Printf("close prepared insert: %v", err)
|
|
|
|
|
|
+ log.Error("close prepared insert: %v", err)
|
|
}
|
|
}
|
|
}()
|
|
}()
|
|
|
|
|
|
// insert records
|
|
// insert records
|
|
tx, err := db.BeginTx(ctx, nil)
|
|
tx, err := db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("begin transaction: %v", err))
|
|
|
|
|
|
+ log.Fatal("begin transaction: %v", err)
|
|
}
|
|
}
|
|
i := 0
|
|
i := 0
|
|
for {
|
|
for {
|
|
@@ -130,11 +137,11 @@ func main() {
|
|
}
|
|
}
|
|
if i&65535 == 0 {
|
|
if i&65535 == 0 {
|
|
if err := tx.Commit(); err != nil {
|
|
if err := tx.Commit(); err != nil {
|
|
- log.Fatal(fmt.Printf("commit transaction: %v", err))
|
|
|
|
|
|
+ log.Fatal("commit transaction: %v", err)
|
|
}
|
|
}
|
|
tx, err = db.BeginTx(ctx, nil)
|
|
tx, err = db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("begin transaction: %v", err))
|
|
|
|
|
|
+ log.Fatal("begin transaction: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -143,9 +150,9 @@ func main() {
|
|
fmt.Println()
|
|
fmt.Println()
|
|
|
|
|
|
if err == io.EOF {
|
|
if err == io.EOF {
|
|
- log.Printf("read %d records", i)
|
|
|
|
|
|
+ fmt.Fprintf(os.Stderr, "read %d records\n", i)
|
|
} else {
|
|
} else {
|
|
- log.Printf("read csv '%s': %v", csvPath, err)
|
|
|
|
|
|
+ log.Error("read csv '%s': %v", csvPath, err)
|
|
}
|
|
}
|
|
break
|
|
break
|
|
}
|
|
}
|
|
@@ -162,11 +169,11 @@ func main() {
|
|
}
|
|
}
|
|
_, err = insert.ExecContext(ctx, args...)
|
|
_, err = insert.ExecContext(ctx, args...)
|
|
if err != nil {
|
|
if err != nil {
|
|
- log.Fatal(fmt.Sprintf("insert record '%#v': %v", rec, err))
|
|
|
|
|
|
+ log.Fatal("insert record '%#v': %v", rec, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
if err := tx.Commit(); err != nil {
|
|
- log.Fatal(fmt.Sprintf("commit transaction: %v", err))
|
|
|
|
|
|
+ log.Fatal("commit transaction: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
@@ -183,6 +190,8 @@ func genInsert(
|
|
name string,
|
|
name string,
|
|
headers []string,
|
|
headers []string,
|
|
) (*sql.Stmt, error) {
|
|
) (*sql.Stmt, error) {
|
|
|
|
+ log := ctx.Value("log").(*logger.Logger)
|
|
|
|
+
|
|
var b strings.Builder
|
|
var b strings.Builder
|
|
|
|
|
|
b.WriteString(fmt.Sprintf("INSERT INTO %s (", name))
|
|
b.WriteString(fmt.Sprintf("INSERT INTO %s (", name))
|
|
@@ -200,7 +209,7 @@ func genInsert(
|
|
b.WriteString(fmt.Sprintf("$%d", i+1))
|
|
b.WriteString(fmt.Sprintf("$%d", i+1))
|
|
}
|
|
}
|
|
b.WriteString(");")
|
|
b.WriteString(");")
|
|
- log.Printf("debug: prepare insert: %s\n", b.String())
|
|
|
|
|
|
+ log.Debug("debug: prepare insert: %s\n", b.String())
|
|
|
|
|
|
return db.PrepareContext(ctx, b.String())
|
|
return db.PrepareContext(ctx, b.String())
|
|
}
|
|
}
|
|
@@ -225,6 +234,8 @@ func createTable(
|
|
}
|
|
}
|
|
b.WriteString(");")
|
|
b.WriteString(");")
|
|
|
|
|
|
|
|
+ fmt.Fprintf(os.Stderr, "%s\n", b.String())
|
|
|
|
+
|
|
return db.ExecContext(ctx, b.String())
|
|
return db.ExecContext(ctx, b.String())
|
|
}
|
|
}
|
|
|
|
|
|
@@ -246,5 +257,5 @@ func scrubName(s string) string {
|
|
s = unders.ReplaceAllLiteralString(s, "_")
|
|
s = unders.ReplaceAllLiteralString(s, "_")
|
|
s = strings.TrimSuffix(s, "_")
|
|
s = strings.TrimSuffix(s, "_")
|
|
|
|
|
|
- return s
|
|
|
|
|
|
+ return fmt.Sprintf("'%s'", s)
|
|
}
|
|
}
|