Skip records with primary key constraint violations

main.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/csv"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -18,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mattn/go-sqlite3"
 	_ "github.com/mattn/go-sqlite3"
 	logger "idio.link/go/logger/v3"
 
@@ -30,6 +32,8 @@ var (
 
 const MaxRecords = 1_000_000_000
 
+const SQLITE_CONSTRAINT_PRIMARYKEY = 1555
+
 func main() {
 	log := logger.NewLogger()
 
@@ -122,6 +126,7 @@ func main() {
 	if err != nil {
 		log.Fatal("begin transaction: %v", err)
 	}
+	nonUnique := 0
 	i := 0
 	for {
 		i++
@@ -151,6 +156,7 @@ func main() {
 
 			if err == io.EOF {
 				fmt.Fprintf(os.Stderr, "read %d records\n", i)
+				fmt.Fprintf(os.Stderr, "skipped %d redundant records\n", nonUnique)
 			} else {
 				log.Error("read csv '%s': %v", csvPath, err)
 			}
@@ -167,9 +173,17 @@ func main() {
 		for j := 0; j < len(rec); j++ {
 			args[j] = any(rec[j])
 		}
+      	// For explanation of extended codes, see https://www.sqlite.org/rescode.html
 		_, err = insert.ExecContext(ctx, args...)
 		if err != nil {
-			log.Fatal("insert record '%#v': %v", rec, err)
+			var sqliteErr sqlite3.Error
+			if errors.As(err, &sqliteErr) &&
+				sqliteErr.ExtendedCode == SQLITE_CONSTRAINT_PRIMARYKEY { // primary key
+				nonUnique++
+				log.Debug("insert record '%#v': %v", rec, err)
+			} else {
+				log.Fatal("insert record '%#v': %v", rec, err)
+			}
 		}
 	}
 	if err := tx.Commit(); err != nil {
@@ -258,4 +272,4 @@ func scrubName(s string) string {
 	s = strings.TrimSuffix(s, "_")
 
 	return fmt.Sprintf("'%s'", s)
-}
+}