0
0
mirror of https://gitlab.com/cznic/sqlite.git synced 2025-04-27 23:07:44 +00:00
Files
go-sqlite/generator.go
2018-11-06 18:03:25 +01:00

698 lines
15 KiB
Go

// Copyright 2017 The Sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
package main
import (
"bytes"
"flag"
"fmt"
"go/format"
"go/scanner"
"go/token"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strings"
"log"
"modernc.org/cc"
"modernc.org/ccgo"
"modernc.org/ccir"
"modernc.org/internal/buffer"
"modernc.org/strutil"
"modernc.org/xc"
)
var (
cpp = flag.Bool("cpp", false, "")
dict = xc.Dict
errLimit = flag.Int("errlimit", 10, "")
filter = flag.String("re", "", "")
ndebug = flag.Bool("ndebug", false, "")
noexec = flag.Bool("noexec", false, "")
oLog = flag.Bool("log", false, "")
trace = flag.Bool("trc", false, "")
unconvertBin string
yydebug = flag.Int("yydebug", 0, "")
)
const (
sqliteRepo = "sqlite.org"
version = "3190300"
prologueSqlite = `// Code generated by ccgo. DO NOT EDIT.
/*
%s
*/
// Package sqlite is an in-process implementation of a self-contained,
// serverless, zero-configuration, transactional SQL database engine. (Work In Progress)
%s
package bin
import (
"fmt"
"math"
"os"
"path"
"runtime"
"unsafe"
"modernc.org/ccgo/crt"
)
func ftrace(s string, args ...interface{}) {
_, fn, fl, _ := runtime.Caller(1)
fmt.Fprintf(os.Stderr, "# %%s:%%d: %%v\n", path.Base(fn), fl, fmt.Sprintf(s, args...))
os.Stderr.Sync()
}
`
prologueTest = `// Code generated by ccgo. DO NOT EDIT.
// %s
%s
package main
import (
"math"
"os"
"unsafe"
"modernc.org/ccgo/crt"
"modernc.org/sqlite/internal/bin"
)
var argv []*int8
func main() {
for _, v := range os.Args {
argv = append(argv, (*int8)(crt.CString(v)))
}
argv = append(argv, nil)
X_start(crt.NewTLS(), int32(len(os.Args)), &argv[0])
}
`
defines = `
#define HAVE_MALLOC_H 1
#define HAVE_MALLOC_USABLE_SIZE 1
#define HAVE_USLEEP 1
#define SQLITE_DEBUG 1
#define SQLITE_ENABLE_API_ARMOR 1
#define SQLITE_USE_URI 1
#define SQLITE_WITHOUT_MSIZE 1
int sqlite3PendingByte;
`
)
func findRepo(s string) string {
s = filepath.FromSlash(s)
for _, v := range strings.Split(strutil.Gopath(), string(os.PathListSeparator)) {
p := filepath.Join(v, "src", s)
fi, err := os.Lstat(p)
if err != nil {
continue
}
if fi.IsDir() {
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
if p, err = filepath.Rel(wd, p); err != nil {
log.Fatal(err)
}
return p
}
}
return ""
}
func errStr(err error) string {
switch x := err.(type) {
case scanner.ErrorList:
if len(x) != 1 {
x.RemoveMultiples()
}
var b bytes.Buffer
for i, v := range x {
if i != 0 {
b.WriteByte('\n')
}
b.WriteString(v.Error())
if i == 9 {
fmt.Fprintf(&b, "\n\t... and %v more errors", len(x)-10)
break
}
}
return b.String()
default:
return err.Error()
}
}
func build(predef string, tus [][]string, ccgoOpts []ccgo.Option, opts ...cc.Opt) ([]*cc.TranslationUnit, []byte) {
ndbg := ""
if *ndebug {
ndbg = "#define NDEBUG 1"
}
var lpos token.Position
if *cpp {
opts = append(opts, cc.Cpp(func(toks []xc.Token) {
if len(toks) != 0 {
p := toks[0].Position()
if p.Filename != lpos.Filename {
fmt.Fprintf(os.Stderr, "# %d %q\n", p.Line, p.Filename)
}
lpos = p
}
for _, v := range toks {
os.Stderr.WriteString(cc.TokSrc(v))
}
os.Stderr.WriteString("\n")
}))
}
var build []*cc.TranslationUnit
tus = append(tus, []string{ccir.CRT0Path})
for _, src := range tus {
model, err := ccir.NewModel()
if err != nil {
log.Fatal(err)
}
ast, err := cc.Parse(
fmt.Sprintf(`
%s
#define _CCGO 1
#define __arch__ %s
#define __os__ %s
#include <builtin.h>
%s
`, ndbg, runtime.GOARCH, runtime.GOOS, predef),
src,
model,
append([]cc.Opt{
cc.AllowCompatibleTypedefRedefinitions(),
cc.EnableEmptyStructs(),
cc.EnableImplicitFuncDef(),
cc.EnableNonConstStaticInitExpressions(),
cc.EnableWideBitFieldTypes(),
cc.ErrLimit(*errLimit),
cc.KeepComments(),
cc.SysIncludePaths([]string{ccir.LibcIncludePath}),
}, opts...)...,
)
if err != nil {
log.Fatal(errStr(err))
}
build = append(build, ast)
}
var out buffer.Bytes
if err := ccgo.New(build, &out, ccgoOpts...); err != nil {
log.Fatal(err)
}
return build, out.Bytes()
}
func macros(buf io.Writer, ast *cc.TranslationUnit) {
fmt.Fprintf(buf, `const (
`)
var a []string
for k, v := range ast.Macros {
if v.Value != nil && v.Type.Kind() != cc.Bool {
switch fn := v.DefTok.Position().Filename; {
case
fn == "builtin.h",
fn == "<predefine>",
strings.HasPrefix(fn, "predefined_"):
// ignore
default:
a = append(a, string(dict.S(k)))
}
}
}
sort.Strings(a)
for _, v := range a {
m := ast.Macros[dict.SID(v)]
if m.Value == nil {
log.Fatal("TODO")
}
switch t := m.Type; t.Kind() {
case
cc.Int, cc.UInt, cc.Long, cc.ULong, cc.LongLong, cc.ULongLong,
cc.Float, cc.Double, cc.LongDouble, cc.Bool:
fmt.Fprintf(buf, "X%s = %v\n", v, m.Value)
case cc.Ptr:
switch t := t.Element(); t.Kind() {
case cc.Char:
fmt.Fprintf(buf, "X%s = %q\n", v, dict.S(int(m.Value.(cc.StringLitID))))
default:
log.Fatalf("%v", t.Kind())
}
default:
log.Fatalf("%v", t.Kind())
}
}
a = a[:0]
for _, v := range ast.Declarations.Identifiers {
switch x := v.Node.(type) {
case *cc.DirectDeclarator:
d := x.TopDeclarator()
id, _ := d.Identifier()
if x.EnumVal == nil {
break
}
a = append(a, string(dict.S(id)))
default:
log.Fatalf("%T", x)
}
}
sort.Strings(a)
for _, v := range a {
dd := ast.Declarations.Identifiers[dict.SID(v)].Node.(*cc.DirectDeclarator)
fmt.Fprintf(buf, "X%s = %v\n", v, dd.EnumVal)
}
fmt.Fprintf(buf, ")\n")
}
func unconvert(pth string) {
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
defer func() {
if err := os.Chdir(wd); err != nil {
log.Fatal(err)
}
}()
if err := os.Chdir(filepath.Dir(pth)); err != nil {
log.Fatal(err)
}
if out, err := exec.Command(unconvertBin, "-apply").CombinedOutput(); err != nil {
log.Fatalf("unconvert: %s\n%s", err, out)
}
}
func cp(dst, src, glob string) {
pat := filepath.Join(filepath.FromSlash(src), glob)
m, err := filepath.Glob(pat)
if err != nil {
log.Fatal(err)
}
if len(m) == 0 {
log.Fatalf("cp(%q, %q, %q): no files for %q", dst, src, glob, pat)
}
dst = filepath.FromSlash(dst)
for _, v := range m {
f, err := ioutil.ReadFile(v)
if err != nil {
log.Fatal(err)
}
_, nm := filepath.Split(v)
if err := ioutil.WriteFile(filepath.Join(dst, nm), f, 0664); err != nil {
log.Fatal(err)
}
}
}
func header(f string) []byte {
b, err := ioutil.ReadFile(f)
if err != nil {
log.Fatal(err)
}
var s scanner.Scanner
s.Init(token.NewFileSet().AddFile(f, -1, len(b)), b, nil, scanner.ScanComments)
var buf buffer.Bytes
for {
_, tok, lit := s.Scan()
switch tok {
case token.COMMENT:
buf.WriteString(lit)
buf.WriteByte('\n')
default:
return buf.Bytes()
}
}
}
func tidyComment(s string) string {
switch {
case strings.HasPrefix(s, "/*"):
a := strings.Split("/"+s[1:len(s)-1], "\n")
for i, v := range a {
a[i] = "// " + v
}
return strings.Join(a, "\n") + "/\n"
case strings.HasPrefix(s, "//"):
return "// " + s[2:] + "\n"
default:
panic("internal error")
}
}
func tidyComments(b []byte) string {
var s scanner.Scanner
s.Init(token.NewFileSet().AddFile("", -1, len(b)), b, nil, scanner.ScanComments)
var a []string
for {
_, tok, lit := s.Scan()
if tok == token.EOF {
return strings.Join(a, "\n")
}
a = append(a, tidyComment(lit))
}
}
func sqlite() {
repo := findRepo(sqliteRepo)
if repo == "" {
log.Fatalf("repository not found: %v", sqliteRepo)
return
}
pth := filepath.Join(repo, "sqlite-amalgamation-"+version)
ast, _ := build(
defines,
[][]string{
{filepath.Join(pth, "sqlite3.h")},
{"main.c"},
},
[]ccgo.Option{ccgo.Library(), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{pth}),
)
sqlite3 := filepath.Join(pth, "sqlite3.c")
_, src := build(
defines,
[][]string{
{sqlite3},
{"main.c"},
},
[]ccgo.Option{ccgo.Library(), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{pth}),
)
var b bytes.Buffer
lic, err := ioutil.ReadFile("SQLITE-LICENSE")
if err != nil {
log.Fatal(err)
}
fmt.Fprintf(&b, prologueSqlite, lic, strings.TrimSpace(tidyComments(header(sqlite3))))
macros(&b, ast[0])
b.Write(src)
b2, err := format.Source(b.Bytes())
if err != nil {
b2 = b.Bytes()
}
if err := os.MkdirAll(filepath.Join("internal", "bin"), 0775); err != nil {
log.Fatal(err)
}
dst := fmt.Sprintf(filepath.Join("internal", "bin", "bin_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
b2 = bytes.Replace(b2, []byte("var Xsqlite3PendingByte int32"), []byte("func Xsqlite3PendingByte() int32 { return _sqlite3PendingByte }"), 1)
if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
log.Fatal(err)
}
unconvert(dst)
}
func mpTest() {
repo := findRepo(sqliteRepo)
if repo == "" {
log.Fatalf("repository not found: %v", sqliteRepo)
return
}
sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
pth := filepath.Join(repo, "sqlite-src-"+version, "mptest")
tag := "mptest"
test := filepath.Join(pth, tag+".c")
_, src := build(
defines,
[][]string{
{filepath.Join(sqlitePth, "sqlite3.c")},
{test},
},
[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{sqlitePth}),
)
var b bytes.Buffer
fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
b.Write(src)
b2, err := format.Source(b.Bytes())
if err != nil {
b2 = b.Bytes()
}
if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
log.Fatal(err)
}
if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
log.Fatal(err)
}
dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
log.Fatal(err)
}
unconvert(dst)
cp(filepath.Join("testdata", tag), pth, "*.test")
cp(filepath.Join("testdata", tag), pth, "*.subtest")
}
func threadTest1() {
repo := findRepo(sqliteRepo)
if repo == "" {
log.Fatalf("repository not found: %v", sqliteRepo)
return
}
sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
tag := "threadtest1"
test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest1.c")
_, src := build(
defines,
[][]string{
{filepath.Join(sqlitePth, "sqlite3.c")},
{test},
},
[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
)
var b bytes.Buffer
fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
b.Write(src)
b2, err := format.Source(b.Bytes())
if err != nil {
b2 = b.Bytes()
}
if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
log.Fatal(err)
}
if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
log.Fatal(err)
}
dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
log.Fatal(err)
}
unconvert(dst)
}
func threadTest2() {
repo := findRepo(sqliteRepo)
if repo == "" {
log.Fatalf("repository not found: %v", sqliteRepo)
return
}
sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
pth := filepath.Join(repo, "sqlite-src-"+version, "test")
tag := "threadtest2"
test := filepath.Join(pth, tag+".c")
_, src := build(
defines,
[][]string{
{filepath.Join(sqlitePth, "sqlite3.c")},
{test},
},
[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
)
var b bytes.Buffer
fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
b.Write(src)
b2, err := format.Source(b.Bytes())
if err != nil {
b2 = b.Bytes()
}
if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
log.Fatal(err)
}
if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
log.Fatal(err)
}
dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
log.Fatal(err)
}
unconvert(dst)
}
func threadTest3() {
n := 3
repo := findRepo(sqliteRepo)
if repo == "" {
log.Fatalf("repository not found: %v", sqliteRepo)
return
}
sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
pth := filepath.Join(repo, "sqlite-src-"+version, "test")
tag := fmt.Sprintf("threadtest%v", n)
test := filepath.Join(pth, tag+".c")
_, src := build(
defines,
[][]string{
{filepath.Join(sqlitePth, "sqlite3.c")},
{filepath.Join(repo, "sqlite-src-"+version, "src", "test_multiplex.c")},
{test},
},
[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
)
var b bytes.Buffer
fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
b.Write(src)
b2, err := format.Source(b.Bytes())
if err != nil {
b2 = b.Bytes()
}
if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
log.Fatal(err)
}
if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
log.Fatal(err)
}
dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
b2 = bytes.Replace(b2, []byte("Xsqlite3PendingByte"), []byte("bin.Xsqlite3PendingByte()"), -1)
if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
log.Fatal(err)
}
unconvert(dst)
}
func threadTest4() {
repo := findRepo(sqliteRepo)
if repo == "" {
log.Fatalf("repository not found: %v", sqliteRepo)
return
}
sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version)
tag := "threadtest4"
test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest4.c")
_, src := build(
defines,
[][]string{
{filepath.Join(sqlitePth, "sqlite3.c")},
{test},
},
[]ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()},
cc.EnableAnonymousStructFields(),
cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}),
)
var b bytes.Buffer
fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test))))
b.Write(src)
b2, err := format.Source(b.Bytes())
if err != nil {
b2 = b.Bytes()
}
if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil {
log.Fatal(err)
}
if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil {
log.Fatal(err)
}
dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH)
if err := ioutil.WriteFile(dst, b2, 0664); err != nil {
log.Fatal(err)
}
unconvert(dst)
}
func main() {
log.SetFlags(log.Lshortfile | log.Lmicroseconds)
var err error
if unconvertBin, err = exec.LookPath("unconvert"); err != nil {
log.Fatal("Please install the unconvert tool (go get -u github.com/mdempsky/unconvert)")
}
flag.Parse()
sqlite()
mpTest()
threadTest1()
threadTest2()
threadTest3()
threadTest4()
}