diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e56fd4adf..3e1fdbdd6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -230,8 +230,27 @@ jobs: # run specialized role based tests make test-authorization ARGS="--grip_config_file_path test/pebble-auth.yml" - - + pygripTest: + needs: build + name: PyGrip UnitTest + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + - name: Python Dependencies for Conformance + run: pip install requests numpy PyYAML pytest jsonpath-ng + - name: install gripql + run: | + cd gripql/python + python setup.py install --user + - name: install pygrip + run: | + python setup.py install --user + - name: unit tests + run: | + cd test + python -m unittest discover -s ./pygrip_test + gridsTest: needs: build name: GRIDs Conformance diff --git a/accounts/basic.go b/accounts/basic.go index ffbf6b119..01c74761a 100644 --- a/accounts/basic.go +++ b/accounts/basic.go @@ -4,6 +4,8 @@ import ( "encoding/base64" "fmt" "strings" + + "github.com/bmeg/grip/log" ) // BasicCredential describes a username and password for use with Funnel's basic auth. @@ -18,7 +20,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { var auth []string var ok bool - fmt.Printf("Running BasicAuth: %#v\n", md) + log.Infof("Running BasicAuth: %#v\n", md) if auth, ok = md["Authorization"]; !ok { if auth, ok = md["authorization"]; !ok { @@ -28,7 +30,7 @@ func (ba BasicAuth) Validate(md MetaData) (string, error) { if len(auth) > 0 { user, password, ok := parseBasicAuth(auth[0]) - fmt.Printf("User: %s Password: %s OK: %s\n", user, password, ok) + log.Debugf("User: %s Password: %s OK: %#v\n", user, password, ok) for _, c := range ba { if c.User == user && c.Password == password { return user, nil diff --git a/accounts/casbin.go b/accounts/casbin.go index ce4a576bd..d953857a1 100644 --- a/accounts/casbin.go +++ b/accounts/casbin.go @@ -3,6 +3,7 @@ package accounts import ( "fmt" + "github.com/bmeg/grip/log" "github.com/casbin/casbin/v2" ) @@ -17,19 +18,19 @@ func (ce *CasbinAccess) init() { if e, err := casbin.NewEnforcer(ce.Model, ce.Policy); err == nil { ce.encforcer = e } else { - fmt.Printf("Casbin Error: %s", err) + log.Errorf("Casbin Error: %s", err) } } } func (ce *CasbinAccess) Enforce(user string, graph string, operation Operation) error { ce.init() - fmt.Printf("Casbin request '%s' '%s' '%s'\n", user, graph, operation) + log.Infof("Casbin request '%s' '%s' '%s'\n", user, graph, operation) if res, err := ce.encforcer.Enforce(user, graph, string(operation)); res { return nil } else if err != nil { - fmt.Printf("casbin error: %s\n", err) + log.Errorf("casbin error: %s\n", err) } - fmt.Printf("Not allowed: '%s' '%s' '%s'\n", user, graph, operation) + log.Errorf("Not allowed: '%s' '%s' '%s'\n", user, graph, operation) return fmt.Errorf("action restricted") } diff --git a/accounts/util.go b/accounts/util.go index c70a5afd7..b93377264 100644 --- a/accounts/util.go +++ b/accounts/util.go @@ -54,11 +54,11 @@ func (c *Config) StreamInterceptor() grpc.StreamServerInterceptor { // using a password stored in the config. func unaryAuthInterceptor(auth Authenticate, access Access) grpc.UnaryServerInterceptor { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - //fmt.Printf("AuthInt: %#v\n", ctx) + //log.Infof("AuthInt: %#v\n", ctx) md, _ := metadata.FromIncomingContext(ctx) - //fmt.Printf("Metadata: %#v\n", md) + //log.Infof("Metadata: %#v\n", md) //omd, _ := metadata.FromOutgoingContext(ctx) - //fmt.Printf("Raw: %#v\n", omd) + //log.Infof("Raw: %#v\n", omd) metaData := MetaData{} for i := range md { @@ -89,10 +89,10 @@ func unaryAuthInterceptor(auth Authenticate, access Access) grpc.UnaryServerInte // using a password stored in the config. func streamAuthInterceptor(auth Authenticate, access Access) grpc.StreamServerInterceptor { return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - //fmt.Printf("Streaming query: %#v\n", info) + //log.Infof("Streaming query: %#v\n", info) md, _ := metadata.FromIncomingContext(ss.Context()) - //fmt.Printf("Metadata: %#v\n", md) + //log.Infof("Metadata: %#v\n", md) metaData := MetaData{} for i := range md { metaData[i] = md[i] diff --git a/cmd/root.go b/cmd/root.go index b2a85c8d4..d2cacc389 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,6 +10,7 @@ import ( "github.com/bmeg/grip/cmd/delete" "github.com/bmeg/grip/cmd/drop" "github.com/bmeg/grip/cmd/dump" + "github.com/bmeg/grip/cmd/embedded" "github.com/bmeg/grip/cmd/erclient" "github.com/bmeg/grip/cmd/info" "github.com/bmeg/grip/cmd/job" @@ -73,6 +74,7 @@ func init() { RootCmd.AddCommand(version.Cmd) RootCmd.AddCommand(kvload.Cmd) RootCmd.AddCommand(delete.Cmd) + RootCmd.AddCommand(embedded.Cmd) } diff --git a/engine/core/util.go b/engine/core/util.go index 2cc616ef3..03a6ff036 100644 --- a/engine/core/util.go +++ b/engine/core/util.go @@ -1,13 +1,5 @@ package core -import ( - "github.com/kr/pretty" -) - -func debug(i ...interface{}) { - pretty.Println(i...) -} - func dedupStringSlice(s []string) []string { seen := make(map[string]struct{}, len(s)) j := 0 diff --git a/grids/new.go b/grids/new.go index 42c52441b..a5f8e6e93 100644 --- a/grids/new.go +++ b/grids/new.go @@ -9,6 +9,7 @@ import ( "github.com/bmeg/benchtop/bsontable" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/log" "github.com/bmeg/grip/timestamp" ) @@ -42,7 +43,7 @@ func (kgraph *GDB) AddGraph(graph string) error { } func newGraph(baseDir, name string) (*Graph, error) { dbPath := filepath.Join(baseDir, name) - fmt.Printf("Creating new GRIDS graph %s\n", name) + log.Infof("Creating new GRIDS graph %s\n", name) // Create directory if it doesn't exist if _, err := os.Stat(dbPath); os.IsNotExist(err) { @@ -78,7 +79,7 @@ func newGraph(baseDir, name string) (*Graph, error) { func getGraph(baseDir, name string) (*Graph, error) { dbPath := filepath.Join(baseDir, name) - fmt.Printf("fetching GRIDS graph %s\n", name) + log.Infof("fetching GRIDS graph %s\n", name) versionPath := filepath.Join(dbPath, "VERSION") file, err := os.Open(versionPath) diff --git a/gripper/server.go b/gripper/server.go index 6feaa2cd4..0f28ff23a 100644 --- a/gripper/server.go +++ b/gripper/server.go @@ -3,9 +3,9 @@ package gripper import ( "context" "fmt" - "log" "net" + "github.com/bmeg/grip/log" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/structpb" ) @@ -36,13 +36,13 @@ func NewSimpleTableServer(dr map[string]Driver) *SimpleTableServicer { func StartServer(port int, serv GRIPSourceServer) { lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port)) if err != nil { - log.Fatalf("failed to listen: %v", err) + log.Errorf("failed to listen: %v", err) } var opts []grpc.ServerOption grpcServer := grpc.NewServer(opts...) RegisterGRIPSourceServer(grpcServer, serv) - fmt.Printf("Starting: %d\n", port) + log.Infof("Starting: %d\n", port) grpcServer.Serve(lis) } @@ -97,7 +97,7 @@ func (st *SimpleTableServicer) GetRowsByID(srv GRIPSource_GetRowsByIDServer) err if err != nil { break } - log.Printf("Request: %s %s", err, req) + log.Debugf("Request: %s %s", err, req) if dr, ok := st.drivers[req.Collection]; ok { if row, err := dr.FetchRow(req.Id); err == nil { data, _ := structpb.NewStruct(row.Value) diff --git a/gripql/marshal_flattened.go b/gripql/marshal_flattened.go index da2ab6bc0..6a285a3ea 100644 --- a/gripql/marshal_flattened.go +++ b/gripql/marshal_flattened.go @@ -2,8 +2,8 @@ package gripql import ( "encoding/json" - "fmt" + "github.com/bmeg/grip/log" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" @@ -80,7 +80,7 @@ func (mflat *MarshalFlatten) Unmarshal(data []byte, v interface{}) error { } s, err := structpb.NewStruct(data) if err != nil { - fmt.Printf("NewStruct error: %s", err) + log.Errorf("NewStruct error: %s", err) } if err == nil { y.Data = s diff --git a/gripql/python/gripql/__init__.py b/gripql/python/gripql/__init__.py index 8cc78e544..bbacd739d 100644 --- a/gripql/python/gripql/__init__.py +++ b/gripql/python/gripql/__init__.py @@ -36,4 +36,4 @@ count ] -__version__ = "0.7.1" +__version__ = "0.8.0" diff --git a/gripql/python/gripql/graph.py b/gripql/python/gripql/graph.py index 6ba38e30c..17dc64078 100644 --- a/gripql/python/gripql/graph.py +++ b/gripql/python/gripql/graph.py @@ -197,6 +197,12 @@ def query(self): """ return Query(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file) + def V(self, *args): + """ + Create a vertex query handle. + """ + return Query(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file).V(*args) + def resume(self, job_id): """ Create a query handle. diff --git a/gripql/python/gripql/query.py b/gripql/python/gripql/query.py index 9c7092eb5..9315fbda9 100644 --- a/gripql/python/gripql/query.py +++ b/gripql/python/gripql/query.py @@ -36,16 +36,12 @@ def _wrap_dict_value(value): return _wrap_value(value, dict) -class Query(BaseConnection): - def __init__(self, url, graph, user=None, password=None, token=None, credential_file=None, resume=None): - super(Query, self).__init__(url, user, password, token, credential_file) - self.url = self.base_url + "/v1/graph/" + graph + "/query" - self.graph = graph +class QueryBuilder: + def __init__(self): self.query = [] - self.resume = resume def __append(self, part): - q = self.__class__(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file, self.resume) + q = self._builder() q.query = self.query[:] q.query.append(part) return q @@ -360,6 +356,20 @@ def to_dict(self): """ return {"query": self.query} + + +class Query(BaseConnection, QueryBuilder): + def __init__(self, url, graph, user=None, password=None, token=None, credential_file=None, resume=None): + super(Query, self).__init__(url, user, password, token, credential_file) + super(QueryBuilder, self).__init__() + self.url = self.base_url + "/v1/graph/" + graph + "/query" + self.graph = graph + self.query = [] + self.resume = resume + + def _builder(self): + return self.__class__(self.base_url, self.graph, self.user, self.password, self.token, self.credential_file, self.resume) + def __iter__(self): return self.__stream() diff --git a/kvi/leveldb/memdb.go b/kvi/leveldb/memdb.go new file mode 100644 index 000000000..66e8c2752 --- /dev/null +++ b/kvi/leveldb/memdb.go @@ -0,0 +1,168 @@ +package leveldb + +import ( + "bytes" + "fmt" + + "github.com/bmeg/grip/kvi" + "github.com/bmeg/grip/log" + "github.com/syndtr/goleveldb/leveldb/comparer" + "github.com/syndtr/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/memdb" +) + +var mem_loaded = kvi.AddKVDriver("memdb", NewMemKVInterface) + +type LevelMemKV struct { + db *memdb.DB +} + +// NewKVInterface creates new LevelDB backed KVInterface at `path` +func NewMemKVInterface(path string, opts kvi.Options) (kvi.KVInterface, error) { + log.Info("Starting In-Memory LevelDB") + db := memdb.New(comparer.DefaultComparer, 1000) + return &LevelMemKV{db: db}, nil +} + +// BulkWrite implements kvi.KVInterface. +func (l *LevelMemKV) BulkWrite(u func(bl kvi.KVBulkWrite) error) error { + ktx := &memIterator{l.db, nil, true, nil, nil} + return u(ktx) +} + +// Close implements kvi.KVInterface. +func (l *LevelMemKV) Close() error { + return nil +} + +// Delete implements kvi.KVInterface. +func (l *LevelMemKV) Delete(key []byte) error { + return l.db.Delete(key) +} + +// DeletePrefix implements kvi.KVInterface. +func (l *LevelMemKV) DeletePrefix(prefix []byte) error { + deleteBlockSize := 10000 + for found := true; found; { + found = false + wb := make([][]byte, 0, deleteBlockSize) + it := l.db.NewIterator(nil) + for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix) && len(wb) < deleteBlockSize-1; it.Next() { + wb = append(wb, copyBytes(it.Key())) + } + it.Release() + for _, i := range wb { + l.db.Delete(i) + found = true + } + } + return nil + +} + +// Get implements kvi.KVInterface. +func (l *LevelMemKV) Get(key []byte) ([]byte, error) { + return l.db.Get(key) +} + +// HasKey implements kvi.KVInterface. +func (l *LevelMemKV) HasKey(key []byte) bool { + _, err := l.db.Get(key) + return err == nil +} + +// Set implements kvi.KVInterface. +func (l *LevelMemKV) Set(key []byte, value []byte) error { + return l.db.Put(key, value) +} + +// Update implements kvi.KVInterface. +func (l *LevelMemKV) Update(func(tx kvi.KVTransaction) error) error { + panic("unimplemented") +} + +// View implements kvi.KVInterface. +func (l *LevelMemKV) View(u func(it kvi.KVIterator) error) error { + it := l.db.NewIterator(nil) + defer it.Release() + lit := memIterator{l.db, it, true, nil, nil} + return u(&lit) +} + +type memIterator struct { + db *memdb.DB + it iterator.Iterator + forward bool + key []byte + value []byte +} + +// Get retrieves the value of key `id` +func (lit *memIterator) Get(id []byte) ([]byte, error) { + return lit.db.Get(id) +} + +func (lit *memIterator) Set(key, val []byte) error { + return lit.db.Put(key, val) +} + +// Key returns the key the iterator is currently pointed at +func (lit *memIterator) Key() []byte { + return lit.key +} + +// Value returns the valud of the iterator is currently pointed at +func (lit *memIterator) Value() ([]byte, error) { + return lit.value, nil +} + +// Next move the iterator to the next key +func (lit *memIterator) Next() error { + var more bool + if lit.forward { + more = lit.it.Next() + } else { + more = lit.it.Prev() + } + if !more { + lit.key = nil + lit.value = nil + return fmt.Errorf("Invalid") + } + lit.key = copyBytes(lit.it.Key()) + lit.value = copyBytes(lit.it.Value()) + return nil +} + +func (lit *memIterator) Seek(id []byte) error { + lit.forward = true + if lit.it.Seek(id) { + lit.key = copyBytes(lit.it.Key()) + lit.value = copyBytes(lit.it.Value()) + return nil + } + return fmt.Errorf("Invalid") +} + +func (lit *memIterator) SeekReverse(id []byte) error { + lit.forward = false + if lit.it.Seek(id) { + //Level iterator will land on the first value above the request + //if we're there, move once to get below start request + if bytes.Compare(id, lit.it.Key()) < 0 { + lit.it.Prev() + } + lit.key = copyBytes(lit.it.Key()) + lit.value = copyBytes(lit.it.Value()) + return nil + } + return fmt.Errorf("Invalid") +} + +// Valid returns true if iterator is still in valid location +func (lit *memIterator) Valid() bool { + if lit.key == nil || lit.value == nil { + return false + } + return true +} diff --git a/main.go b/main.go index df320e840..53fa06cea 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "os" "github.com/bmeg/grip/cmd" @@ -11,7 +10,7 @@ import ( func main() { log.ConfigureLogger(log.DefaultLoggerConfig()) if err := cmd.RootCmd.Execute(); err != nil { - fmt.Println("Error:", err.Error()) + log.Errorf("Error:", err.Error()) os.Exit(1) } } diff --git a/mongo/compile.go b/mongo/compile.go index 2d2eb382a..29391fe17 100644 --- a/mongo/compile.go +++ b/mongo/compile.go @@ -528,10 +528,10 @@ func (comp *Compiler) Compile(stmts []*gripql.GraphStatement, opts *gdbi.Compile keys := protoutil.AsStringList(stmt.HasKey) for _, key := range keys { lKey := ToPipelinePath(key) - fmt.Printf("Key: %s -> %s\n", key, lKey) + //log.Debugf("Key: %s -> %s\n", key, lKey) hasKeys[lKey] = bson.M{"$exists": true} } - fmt.Printf("hasKey: %#v\n", hasKeys) + //log.Debugf("hasKey: %#v\n", hasKeys) query = append(query, bson.D{primitive.E{Key: "$match", Value: hasKeys}}) case *gripql.GraphStatement_Limit: @@ -581,7 +581,7 @@ func (comp *Compiler) Compile(stmts []*gripql.GraphStatement, opts *gdbi.Compile }, }, }) - fmt.Printf("Distinct: %s\n", query) + //log.Debugf("Distinct: %s\n", query) switch lastType { case gdbi.VertexData: query = append(query, bson.D{primitive.E{Key: "$project", Value: bson.M{ diff --git a/mongo/has_evaluator.go b/mongo/has_evaluator.go index 5e08af15f..c4f644c57 100644 --- a/mongo/has_evaluator.go +++ b/mongo/has_evaluator.go @@ -1,8 +1,6 @@ package mongo import ( - "fmt" - "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "go.mongodb.org/mongo-driver/bson" @@ -22,7 +20,7 @@ func convertHasExpression(stmt *gripql.HasExpression, not bool) bson.M { } else { key := cond.Key output = convertHasExpression(gripql.And(gripql.Gt(key, lims[0]), gripql.Lt(key, lims[1])), not) - fmt.Printf("inside: %#v\n", output) + //log.Debugf("inside: %#v\n", output) } case gripql.Condition_OUTSIDE: diff --git a/pygrip/Makefile b/pygrip/Makefile new file mode 100644 index 000000000..dbe3ccf27 --- /dev/null +++ b/pygrip/Makefile @@ -0,0 +1,4 @@ + + +pygrip/gripql_pb2.py: ../gripql/gripql.proto + protoc --proto_path=../gripql/ --python_out=./pygrip -I ../googleapis/ ../gripql/gripql.proto \ No newline at end of file diff --git a/pygrip/pygrip.h b/pygrip/pygrip.h new file mode 100644 index 000000000..db6abcf1c --- /dev/null +++ b/pygrip/pygrip.h @@ -0,0 +1,86 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern GoUintptr NewMemServer(); +extern void AddVertex(GoUintptr graph, GoString gid, GoString label, GoString jdata); +extern void AddEdge(GoUintptr graph, GoString gid, GoString src, GoString dst, GoString label, GoString jdata); +extern GoUintptr Query(GoUintptr graph, GoString jquery); +extern GoUint8 ReaderDone(GoUintptr reader); +extern char* ReaderNext(GoUintptr reader); + +#ifdef __cplusplus +} +#endif diff --git a/pygrip/wrapper.go b/pygrip/wrapper.go new file mode 100644 index 000000000..de9f7bbaa --- /dev/null +++ b/pygrip/wrapper.go @@ -0,0 +1,175 @@ +package main + +/* +#include // for uintptr_t +*/ + +import "C" + +import ( + "context" + "encoding/json" + + "runtime/cgo" + + "github.com/bmeg/grip/engine" + "github.com/bmeg/grip/engine/core" + "github.com/bmeg/grip/engine/pipeline" + "github.com/bmeg/grip/gdbi" + "github.com/bmeg/grip/grids" + "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/kvgraph" + "github.com/bmeg/grip/kvi" + "github.com/bmeg/grip/kvi/leveldb" + "github.com/bmeg/grip/log" + "google.golang.org/protobuf/encoding/protojson" +) + +var graphDB gdbi.GraphDB + +type GraphHandle uintptr +type QueryReaderHandle uintptr + +type Reader interface { + Done() bool + Next() string +} + +//export NewMemServer +func NewMemServer() GraphHandle { + db, _ := leveldb.NewMemKVInterface("", kvi.Options{}) + graphDB = kvgraph.NewKVGraph(db) + err := graphDB.AddGraph("default") + if err != nil { + log.Errorf("Graph init error: %s\n", err) + } + g, err := graphDB.Graph("default") + if err != nil { + log.Errorf("Graph init error: %s\n", err) + } + return GraphHandle(cgo.NewHandle(g)) +} + +func NewGRIDServer(path string) GraphHandle { + graphDB, err := grids.NewGraphDB(path) + if err != nil { + log.Errorf("Graph init error: %s\n", err) + } + g, err := graphDB.Graph("default") + if err != nil { + log.Errorf("Graph init error: %s\n", err) + } + return GraphHandle(cgo.NewHandle(g)) +} + +func CloseServer(graph GraphHandle) { + cgo.Handle(graph).Delete() +} + +//export AddVertex +func AddVertex(graph GraphHandle, gid, label, jdata string) { + data := map[string]any{} + err := json.Unmarshal([]byte(jdata), &data) + if err != nil { + log.Errorf("Data error: %s : %s\n", err, jdata) + } + + g := cgo.Handle(graph).Value().(gdbi.GraphInterface) + + g.AddVertex([]*gdbi.Vertex{ + {ID: gid, Label: label, Data: data}, + }) +} + +//export AddEdge +func AddEdge(graph GraphHandle, gid, src, dst, label, jdata string) { + data := map[string]any{} + err := json.Unmarshal([]byte(jdata), &data) + if err != nil { + log.Errorf("Data error: %s : %s\n", err, jdata) + } + + g := cgo.Handle(graph).Value().(gdbi.GraphInterface) + + g.AddEdge([]*gdbi.Edge{ + {ID: gid, To: dst, From: src, Label: label, Data: data}, + }) +} + +type QueryReader struct { + pipe gdbi.Pipeline + results chan *gripql.QueryResult + current *gripql.QueryResult +} + +//export Query +func Query(graph GraphHandle, jquery string) QueryReaderHandle { + query := gripql.GraphQuery{} + err := protojson.Unmarshal([]byte(jquery), &query) + if err != nil { + log.Errorf("Query error: %s : %s\n", err) + } + + g := cgo.Handle(graph).Value().(gdbi.GraphInterface) + compiler := core.NewCompiler(g) + pipe, err := compiler.Compile(query.Query, nil) + if err != nil { + log.Errorf("Compile error: %s : %s\n", err) + } + + ctx := context.Background() + + bufsize := 5000 + resch := make(chan *gripql.QueryResult, bufsize) + go func() { + defer close(resch) + graph := pipe.Graph() + dataType := pipe.DataType() + markTypes := pipe.MarkTypes() + man := engine.NewManager("./") //TODO: in memory option + rPipe := pipeline.Start(ctx, pipe, man, bufsize, nil, nil) + for t := range rPipe.Outputs { + if !t.IsSignal() { + resch <- pipeline.Convert(graph, dataType, markTypes, t) + } + } + man.Cleanup() + }() + var o = &QueryReader{ + pipe: pipe, + results: resch, + current: nil, + } + return QueryReaderHandle(cgo.NewHandle(o)) +} + +//export ReaderDone +func ReaderDone(reader QueryReaderHandle) bool { + r := cgo.Handle(reader).Value().(*QueryReader) + return r.Done() +} + +//export ReaderNext +func ReaderNext(reader QueryReaderHandle) *C.char { + r := cgo.Handle(reader).Value().(*QueryReader) + o := r.Next() + return C.CString(o) +} + +func (r *QueryReader) Next() string { + out, _ := protojson.Marshal(r.current) + return string(out) +} + +func (r *QueryReader) Done() bool { + select { + case i, ok := <-r.results: + if ok { + r.current = i + return false + } + return true + } +} + +func main() {} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..fdfcb6508 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath = . gripql/python grip diff --git a/server/api.go b/server/api.go index 44d548523..4c7a9cff2 100644 --- a/server/api.go +++ b/server/api.go @@ -663,7 +663,7 @@ func (server *GripServer) AddSchema(ctx context.Context, req *gripql.Graph) (*gr func (server *GripServer) AddJsonSchema(ctx context.Context, rawjson *gripql.RawJson) (*gripql.EditResult, error) { bytes, err := protojson.Marshal(rawjson.Data) if err != nil { - fmt.Printf("Failed to marshal data to bytes: %v\n", err) + log.Errorf("Failed to marshal data to bytes: %v\n", err) return nil, err } req, err := schema.ParseJSchema(bytes, rawjson.Graph) diff --git a/server/endpoints.go b/server/endpoints.go index 4adff8840..490e3d5ee 100644 --- a/server/endpoints.go +++ b/server/endpoints.go @@ -25,7 +25,7 @@ func (server *GripServer) AddEndpoint(name string, path string, config map[strin if err != nil { return err } - fmt.Printf("Method: %#v\n", gen) + log.Infof("Method: %#v\n", gen) if x, ok := (gen).(func(client gripql.Client, config map[string]string) (http.Handler, error)); ok { log.Infof("Plugin %s loaded", path) endpointMap[name] = x diff --git a/server/server.go b/server/server.go index 02a191655..fa2c61309 100644 --- a/server/server.go +++ b/server/server.go @@ -122,7 +122,7 @@ func NewGripServer(conf *config.Config, baseDir string, drivers map[string]gdbi. if _, ok := gdbs[conf.Default]; !ok { return nil, fmt.Errorf("default driver '%s' does not exist", conf.Default) } - fmt.Printf("Default graph driver: %s\n", conf.Default) + log.Infof("Default graph driver: %s\n", conf.Default) return server, nil } diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..4dba5c58c --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +"""Setup for checksig package""" +import sys +from distutils.errors import CompileError +from subprocess import call + +from setuptools import Extension, setup, find_packages +from setuptools.command.build_ext import build_ext + + +class build_go_ext(build_ext): + """Custom command to build extension from Go source files""" + def build_extension(self, ext): + ext_path = self.get_ext_fullpath(ext.name) + cmd = ['go', 'build', '-buildmode=c-shared', '-o', ext_path] + cmd += ext.sources + out = call(cmd) + if out != 0: + raise CompileError('Go build failed') + +setup( + name='pygrip', + version='0.8.0', + packages=find_packages(include=['pygrip']), + #py_modules=['pygrip'], + ext_modules=[ + Extension('pygrip/_pygrip', ['./pygrip/wrapper.go']) + ], + cmdclass={'build_ext': build_go_ext}, + install_requires=[ + "gripql>=0.8.0" + ], + zip_safe=False, +) \ No newline at end of file diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/pygrip_test/__init__.py b/test/pygrip_test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/pygrip_test/fhir/README.md b/test/pygrip_test/fhir/README.md new file mode 100644 index 000000000..c41134976 --- /dev/null +++ b/test/pygrip_test/fhir/README.md @@ -0,0 +1,5 @@ +This test has a dependency +```commandline +pip install jsonpath-ng + +``` \ No newline at end of file diff --git a/test/pygrip_test/fhir/__init__.py b/test/pygrip_test/fhir/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/DocumentReference.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/DocumentReference.ndjson new file mode 100644 index 000000000..8da14c2b2 --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/DocumentReference.ndjson @@ -0,0 +1 @@ +{"resourceType":"DocumentReference","id":"9ae7e542-767f-4b03-a854-7ceed17152cb","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"9ae7e542-767f-4b03-a854-7ceed17152cb"}],"status":"current","docStatus":"final","subject":{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"},"date":"2024-08-21T10:53:00+00:00","content":[{"attachment":{"extension":[{"url":"http://aced-idp.org/fhir/StructureDefinition/md5","valueString":"227f0a5379362d42eaa1814cfc0101b8"},{"url":"http://aced-idp.org/fhir/StructureDefinition/source_path","valueUrl":"file:///home/LabA/specimen_1234_labA.fq.gz"}],"contentType":"text/fastq","url":"file:///home/LabA/specimen_1234_labA.fq.gz","size":5595609484,"title":"specimen_1234_labA.fq.gz","creation":"2024-08-21T10:53:00+00:00"}}]} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Observation.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Observation.ndjson new file mode 100644 index 000000000..774b7051d --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Observation.ndjson @@ -0,0 +1,3 @@ +{"resourceType":"Observation","id":"cec32723-9ede-5f24-ba63-63cb8c6a02cf","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"patientX_1234-9ae7e542-767f-4b03-a854-7ceed17152cb-sequencer"}], "status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"Laboratory"}]}],"code":{"coding":[{"system":"https://my_demo.org/labA","code":"Gen3 Sequencing Metadata","display":"Gen3 Sequencing Metadata"}]},"subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"},"focus":[{"reference":"DocumentReference/9ae7e542-767f-4b03-a854-7ceed17152cb"}],"specimen":{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"},"component":[{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"sequencer","display":"sequencer"}],"text":"sequencer"},"valueString":"Illumina Seq 1000"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"index","display":"index"}],"text":"index"},"valueString":"100bp Single index"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"type","display":"type"}],"text":"type"},"valueString":"Exome"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"project_id","display":"project_id"}],"text":"project_id"},"valueString":"labA_projectXYZ"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"read_length","display":"read_length"}],"text":"read_length"},"valueString":"100"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"instrument_run_id","display":"instrument_run_id"}],"text":"instrument_run_id"},"valueString":"234_ABC_1_8899"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"capture_bait_set","display":"capture_bait_set"}],"text":"capture_bait_set"},"valueString":"Human Exom 2X"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"end_type","display":"end_type"}],"text":"end_type"},"valueString":"Paired-End"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"capture","display":"capture"}],"text":"capture"},"valueString":"emitter XT"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"sequencing_site","display":"sequencing_site"}],"text":"sequencing_site"},"valueString":"AdvancedGeneExom"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"construction","display":"construction"}],"text":"construction"},"valueString":"library_construction"}]} +{"resourceType":"Observation","id":"4e3c6b59-b1fd-5c26-a611-da4cde9fd061","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"patientX_1234-specimen_1234_labA-sample_type"}],"status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"Laboratory"}],"text":"Laboratory"}],"code":{"coding":[{"system":"https://my_demo.org/labA","code":"labA specimen metadata","display":"labA specimen metadata"}],"text":"sample type abc"},"subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"},"focus":[{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"}],"component":[{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"sample_type","display":"sample_type"}],"text":"sample_type"},"valueString":"Primary Solid Tumor"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"library_id","display":"library_id"}],"text":"library_id"},"valueString":"12345"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"tissue_type","display":"tissue_type"}],"text":"tissue_type"},"valueString":"Tumor"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"treatments","display":"treatments"}],"text":"treatments"},"valueString":"Trastuzumab"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"allocated_for_site","display":"allocated_for_site"}],"text":"allocated_for_site"},"valueString":"TEST Clinical Research"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"pathology_data","display":"pathology_data"}],"text":"pathology_data"}},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"clinical_event","display":"clinical_event"}],"text":"clinical_event"}},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"indexed_collection_date","display":"indexed_collection_date"}],"text":"indexed_collection_date"},"valueInteger":365},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"biopsy_specimens_bems_id","display":"biopsy_specimens_bems_id"}],"text":"biopsy_specimens"},"valueString":"specimenA, specimenB, specimenC"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"biopsy_procedure_type","display":"biopsy_procedure_type"}],"text":"biopsy_procedure_type"},"valueString":"Biopsy - Core"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"biopsy_anatomical_location","display":"biopsy_anatomical_location"}],"text":"biopsy_anatomical_location"},"valueString":"top axillary lymph node"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"percent_tumor","display":"percent_tumor"}],"text":"percent_tumor"},"valueString":"30"}]} +{"resourceType":"Observation","id":"21f3411d-89a4-4bcc-9ce7-b76edb1c745f","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"patientX_1234-9ae7e542-767f-4b03-a854-7ceed17152cb-Gene"}], "status":"final","category":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/observation-category","code":"laboratory","display":"Laboratory"}]}],"code":{"coding":[{"system":"https://loinc.org","code":"81247-9","display":"Genomic structural variant copy number"}]},"subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"},"focus":[{"reference":"DocumentReference/9ae7e542-767f-4b03-a854-7ceed17152cb"}],"specimen":{"reference":"Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9"},"component":[{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"Gene","display":"Gene"}],"text":"Gene"},"valueString":"TP53"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"Chromosome","display":"Chromosome"}],"text":"Chromosome"},"valueString":"chr17"},{"code":{"coding":[{"system":"https://my_demo.org/labA","code":"result","display":"result"}],"text":"result"},"valueString":"gain of function (GOF)"}]} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Organization.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Organization.ndjson new file mode 100644 index 000000000..967445ae7 --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Organization.ndjson @@ -0,0 +1 @@ +{"resourceType":"Organization","id":"89c8dc4c-2d9c-48c7-8862-241a49a78f14","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"LabA_ORGANIZATION"}],"type":[{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/organization-type","code":"prov","display":"Healthcare Provider"}],"text":"An organization that provides healthcare services."},{"coding":[{"system":"http://terminology.hl7.org/CodeSystem/organization-type","code":"edu","display":"Educational Institute"}],"text":"An educational institution that provides education or research facilities."}]} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Patient.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Patient.ndjson new file mode 100644 index 000000000..107bf78e5 --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Patient.ndjson @@ -0,0 +1 @@ +{"resourceType":"Patient","id":"bc4e1aa6-cb52-40e9-8f20-594d9c84f920","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"patientX_1234"}],"active":true} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/ResearchStudy.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/ResearchStudy.ndjson new file mode 100644 index 000000000..74cc40029 --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/ResearchStudy.ndjson @@ -0,0 +1 @@ +{"resourceType":"ResearchStudy","id":"7dacd4d0-3c8e-470b-bf61-103891627d45","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"labA"}],"name":"LabA","status":"active","description":"LabA Clinical Trial Study: FHIR Schema Chorot Integration"} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/ResearchSubject.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/ResearchSubject.ndjson new file mode 100644 index 000000000..6aee6d082 --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/ResearchSubject.ndjson @@ -0,0 +1 @@ +{"resourceType":"ResearchSubject","id":"2fc448d6-a23b-4b94-974b-c66110164851","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"subjectX_1234"}],"status":"active","study":{"reference":"ResearchStudy/7dacd4d0-3c8e-470b-bf61-103891627d45"},"subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"}} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Specimen.ndjson b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Specimen.ndjson new file mode 100644 index 000000000..b79c72cbf --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/META/Specimen.ndjson @@ -0,0 +1 @@ +{"resourceType":"Specimen","id":"60c67a06-ea2d-4d24-9249-418dc77a16a9","identifier":[{"use":"official","system":"https://my_demo.org/labA","value":"specimen_1234_labA"}],"subject":{"reference":"Patient/bc4e1aa6-cb52-40e9-8f20-594d9c84f920"},"collection":{"collector":{"reference":"Organization/89c8dc4c-2d9c-48c7-8862-241a49a78f14"},"bodySite":{"concept":{"coding":[{"system":"http://snomed.info/sct","code":"76752008","display":"Breast"}],"text":"Breast"}}},"processing":[{"method":{"coding":[{"system":"http://snomed.info/sct","code":"117032008","display":"Spun specimen (procedure)"},{"system":"https://my_demo.org/labA","code":"Double-Spun","display":"Double-Spun"}],"text":"Spun specimen (procedure)"}}]} \ No newline at end of file diff --git a/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/README.md b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/README.md new file mode 100644 index 000000000..bcad0e36e --- /dev/null +++ b/test/pygrip_test/fhir/fixtures/fhir-compbio-examples/README.md @@ -0,0 +1,11 @@ +##### META folder test-data: + +``` +>>>> resources={'summary': {'DocumentReference': 1, 'Specimen': 1, 'Observation': 3, 'ResearchStudy': 1, 'ResearchSubject': 1, 'Organization': 1, 'Patient': 1}} +``` + +There are three Observations with user-defined metadata component. +1. Focus - reference -> Specimen +2. Focus - reference -> DocumentReference + 1. The first Observation contains metadata on the file's sequencing metadata. + 2. The second Observation includes a simple summary of a CNV analysis result computed from this file. diff --git a/test/pygrip_test/fhir/test_load.py b/test/pygrip_test/fhir/test_load.py new file mode 100644 index 000000000..8ce6e087c --- /dev/null +++ b/test/pygrip_test/fhir/test_load.py @@ -0,0 +1,271 @@ +import json +import pathlib +import types +from collections import defaultdict + +import pytest + +import pygrip.pygrip as pygrip +from jsonpath_ng import jsonpath, parse + +from typing import Generator, Dict, Any + + +def resources() -> Generator[Dict[str, Any], None, None]: + """Read a directory of ndjson files, return dictionary for each line.""" + base = pathlib.Path(__file__).parent.absolute() + fixture_path = pathlib.Path(base / 'fixtures' / 'fhir-compbio-examples' / 'META') + assert fixture_path.exists(), f"Fixture path {fixture_path.absolute()} does not exist." + for file in fixture_path.glob('*.ndjson'): + with open(str(file)) as fp: + for l_ in fp.readlines(): + yield json.loads(l_) + + +@pytest.fixture +def expected_edges() -> list[tuple]: + """Return the expected edges for the resources [(src, dst, label)].""" + return [('21f3411d-89a4-4bcc-9ce7-b76edb1c745f', '60c67a06-ea2d-4d24-9249-418dc77a16a9', 'specimen'), + ('21f3411d-89a4-4bcc-9ce7-b76edb1c745f', '9ae7e542-767f-4b03-a854-7ceed17152cb', 'focus'), + ('21f3411d-89a4-4bcc-9ce7-b76edb1c745f', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920', 'subject'), + ('2fc448d6-a23b-4b94-974b-c66110164851', '7dacd4d0-3c8e-470b-bf61-103891627d45', 'study'), + ('2fc448d6-a23b-4b94-974b-c66110164851', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920', 'subject'), + ('4e3c6b59-b1fd-5c26-a611-da4cde9fd061', '60c67a06-ea2d-4d24-9249-418dc77a16a9', 'focus'), + ('4e3c6b59-b1fd-5c26-a611-da4cde9fd061', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920', 'subject'), + ('60c67a06-ea2d-4d24-9249-418dc77a16a9', '89c8dc4c-2d9c-48c7-8862-241a49a78f14', 'collection_collector'), + ('60c67a06-ea2d-4d24-9249-418dc77a16a9', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920', 'subject'), + ('9ae7e542-767f-4b03-a854-7ceed17152cb', '60c67a06-ea2d-4d24-9249-418dc77a16a9', 'subject'), + ('cec32723-9ede-5f24-ba63-63cb8c6a02cf', '60c67a06-ea2d-4d24-9249-418dc77a16a9', 'specimen'), + ('cec32723-9ede-5f24-ba63-63cb8c6a02cf', '9ae7e542-767f-4b03-a854-7ceed17152cb', 'focus'), + ('cec32723-9ede-5f24-ba63-63cb8c6a02cf', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920', 'subject')] + + +@pytest.fixture +def expected_vertices() -> list[tuple]: + """Return the expected vertices [(id, label)] for the resources.""" + return [('21f3411d-89a4-4bcc-9ce7-b76edb1c745f', 'Observation'), + ('2fc448d6-a23b-4b94-974b-c66110164851', 'ResearchSubject'), + ('4e3c6b59-b1fd-5c26-a611-da4cde9fd061', 'Observation'), + ('60c67a06-ea2d-4d24-9249-418dc77a16a9', 'Specimen'), + ('7dacd4d0-3c8e-470b-bf61-103891627d45', 'ResearchStudy'), + ('89c8dc4c-2d9c-48c7-8862-241a49a78f14', 'Organization'), + ('9ae7e542-767f-4b03-a854-7ceed17152cb', 'DocumentReference'), + ('bc4e1aa6-cb52-40e9-8f20-594d9c84f920', 'Patient'), + ('cec32723-9ede-5f24-ba63-63cb8c6a02cf', 'Observation')] + + +@pytest.fixture +def expected_dataframe_associations(): + """Return the expected dataframe associations for the resources. { (resource_type, resource_id): [(association_resource_type, association_resource_id)].""" + return { + ('ResearchSubject', '2fc448d6-a23b-4b94-974b-c66110164851'): [ + ('ResearchStudy', '7dacd4d0-3c8e-470b-bf61-103891627d45'), + ('Patient', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920'), + ('Specimen', '60c67a06-ea2d-4d24-9249-418dc77a16a9')], + ('Specimen', '60c67a06-ea2d-4d24-9249-418dc77a16a9'): [ + ('ResearchStudy', '7dacd4d0-3c8e-470b-bf61-103891627d45'), + ('ResearchSubject', '2fc448d6-a23b-4b94-974b-c66110164851'), + ('Patient', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920'), + ('Observation', '4e3c6b59-b1fd-5c26-a611-da4cde9fd061')], + ('ResearchStudy', '7dacd4d0-3c8e-470b-bf61-103891627d45'): [ + ('ResearchSubject', '2fc448d6-a23b-4b94-974b-c66110164851')], + ('Organization', '89c8dc4c-2d9c-48c7-8862-241a49a78f14'): [ + ('ResearchStudy', '7dacd4d0-3c8e-470b-bf61-103891627d45'), + ('ResearchSubject', '2fc448d6-a23b-4b94-974b-c66110164851'), + ('Patient', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920'), + ('Specimen', '60c67a06-ea2d-4d24-9249-418dc77a16a9'), + ('DocumentReference', '9ae7e542-767f-4b03-a854-7ceed17152cb')], + ('DocumentReference', '9ae7e542-767f-4b03-a854-7ceed17152cb'): [ + ('ResearchStudy', '7dacd4d0-3c8e-470b-bf61-103891627d45'), + ('ResearchSubject', '2fc448d6-a23b-4b94-974b-c66110164851'), + ('Patient', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920'), + ('Specimen', '60c67a06-ea2d-4d24-9249-418dc77a16a9'), + ('Observation', '21f3411d-89a4-4bcc-9ce7-b76edb1c745f'), + ('Observation', 'cec32723-9ede-5f24-ba63-63cb8c6a02cf')], + ('Patient', 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920'): [ + ('ResearchStudy', '7dacd4d0-3c8e-470b-bf61-103891627d45'), + ('ResearchSubject', '2fc448d6-a23b-4b94-974b-c66110164851'), + ('Specimen', '60c67a06-ea2d-4d24-9249-418dc77a16a9'), + ('Observation', '21f3411d-89a4-4bcc-9ce7-b76edb1c745f'), + ('Observation', '4e3c6b59-b1fd-5c26-a611-da4cde9fd061'), + ('Observation', 'cec32723-9ede-5f24-ba63-63cb8c6a02cf')] + } + + +def match_label(self, vertex_gid, label, seen_already=None) -> dict: + """Recursively find the first vertex of a given label, starting traversals from vertex_gid.""" + + # check params + assert vertex_gid is not None, "Expected vertex_gid to be not None." + assert label is not None, "Expected label to be not None." + # mutable default arguments are evil + # See https://florimond.dev/en/posts/2018/08/python-mutable-defaults-are-the-source-of-all-evil + if seen_already is None: + seen_already = [] + + # get all edges for vertex + q = self.V(vertex_gid).both() + + # get all vertices for edges + # TODO - consider if this should be a vertices_of_label() -> generator[dict] instead + for _ in q: + if _['vertex']['label'] == label: + return _ + else: + if _['vertex']['gid'] in seen_already: + continue + seen_already.append(_['vertex']['gid']) + return self.match_label(_['vertex']['gid'], label, seen_already=seen_already) + + +def dataframe_associations(self, vertex_gid, vertex_label, labels=('ResearchStudy', 'ResearchSubject', 'Patient', 'Specimen', 'DocumentReference', 'Observation')) -> list[dict]: + """Return all objects associated with vertex_gid.""" + associations = [] + for label in labels: + if label == 'Observation': + continue + if vertex_label == label: + continue + _ = self.match_label(vertex_gid, label) + if _ is not None: + associations.append(_['vertex']['data']) + if 'Observation' in labels: + q = self.V(vertex_gid).in_(["focus", "subject"]).hasLabel("Observation") + for _ in q: + associations.append(_['vertex']['data']) + return associations + + +@pytest.fixture +def graph() -> pygrip.GraphDBWrapper: + """Load the resources into the graph. Note: this does _not_ consider iceberg schema.""" + # TODO - add parameter or test environment variable to switch between in-memory and remote graph + graph = pygrip.NewMemServer() + # use jsonpath to find all references with a resource + jsonpath_expr = parse('*..reference') + for _ in resources(): + graph.addVertex(_['id'], _['resourceType'], _) + for match in jsonpath_expr.find(_): + # value will be something like "Specimen/60c67a06-ea2d-4d24-9249-418dc77a16a9" + # full_path will be something like "specimen.reference" or "focus.[0].reference" + type_, dst_id = match.value.split('/') + # determine label from full path + path_parts = str(match.full_path).split('.') + # strip out array indices and reference + path_parts = [part for part in path_parts if '[' not in part and part != 'reference'] + # make it a label + label = '_'.join(path_parts) + graph.addEdge(_['id'], dst_id, label) + + # monkey patch the graph object with our methods + # TODO - consider a more formal subclass of pygrip.GraphDBWrapper + graph.match_label = types.MethodType(match_label, graph) + graph.dataframe_associations = types.MethodType(dataframe_associations, graph) + + yield graph + + +def test_graph_vertices(graph, expected_vertices): + """Test the graph vertices.""" + + actual_vertices = [] + for _ in graph.V(): + assert 'vertex' in _, f"Expected 'vertex' in {_}" + vertex = _['vertex'] + assert 'data' in vertex, f"Expected 'data' in {vertex}" + assert 'gid' in vertex, f"Expected 'gid' in {vertex}" + assert 'label' in vertex, f"Expected 'label' in {vertex}" + assert 'data' in vertex, f"Expected 'data' in {vertex}" + resource = _['vertex']['data'] + actual_vertices.append((resource['id'], resource['resourceType'])) + + print(actual_vertices) + assert actual_vertices == expected_vertices, f"Expected {expected_vertices} but got {actual_vertices}." + + +def test_graph_edges(graph, expected_edges): + """Test the graph vertices.""" + + # check edges all edges + actual_edges = [] + for _ in graph.V().outE(): + assert 'edge' in _, f"Expected 'edge' in {_}" + edge = _['edge'] + assert 'gid' in edge, f"Expected 'gid' in {edge}" + assert 'label' in edge, f"Expected 'label' in {edge}" + assert 'from' in edge, f"Expected 'from' in {edge}" + assert 'to' in edge, f"Expected 'to' in {edge}" + assert 'data' in edge, f"Expected 'data' in {edge}" + + actual_edges.append((edge['from'], edge['to'], edge['label'])) + + print(actual_edges) + assert actual_edges == expected_edges, f"Expected {expected_edges} but got {actual_edges}." + + +def test_graph_methods(graph): + """Test the methods we expect in a graph object.""" + assert 'V' in dir(graph), f"Expected 'V' in {type(graph)}" + assert 'match_label' in dir(graph), f"Expected 'match_label' in {type(graph)}" + assert 'dataframe_associations' in dir(graph), f"Expected 'dataframe_associations' in {type(graph)}" + + +def test_traversals(graph): + """Test basic traversals""" + + # specimen -> patient + q = graph.V().hasLabel("Specimen").out("subject") + actual_specimen_patient_count = len(list(q)) + assert actual_specimen_patient_count == 1, f"Expected 1 but got {actual_specimen_patient_count}." + assert list(q)[0]['vertex']['data']['resourceType'] == 'Patient' + + q = graph.V().hasLabel("DocumentReference").outV().hasLabel("Specimen").outV().hasLabel("Patient") + assert len(list(q)) == 1, f"Expected 1 but got {len(list(q))}." + actual_document_reference_patient_count = len(list(q)) + assert actual_document_reference_patient_count == 1, f"Expected 1 but got {actual_document_reference_patient_count}." + assert list(q)[0]['vertex']['data']['resourceType'] == 'Patient' + + # follow edges by edge label + q = graph.V().hasLabel("DocumentReference").out("subject") + assert len(list(q)) == 1, f"Expected 1 but got {len(list(q))}." + for subject in q: + subject = subject['vertex']['data'] + assert subject['resourceType'] == 'Specimen', f"Expected Specimen but got {subject['resourceType']}." + + # follow all out all edges recursively to a vertex of type X + + q = graph.V().hasLabel("DocumentReference") + assert len(list(q)) == 1, f"Expected 1 but got {len(list(q))}." + document_reference_gid = list(q)[0]['vertex']['gid'] + + # 1 hop + specimen = graph.match_label(document_reference_gid, 'Specimen') + assert specimen is not None, "Expected Specimen" + assert specimen['vertex']['gid'] == '60c67a06-ea2d-4d24-9249-418dc77a16a9', f"Expected 60c67a06-ea2d-4d24-9249-418dc77a16a9 but got {specimen}." + + # 2 hops + patient = graph.match_label(document_reference_gid, 'Patient') + assert patient is not None, "Expected Patient" + assert patient['vertex']['gid'] == 'bc4e1aa6-cb52-40e9-8f20-594d9c84f920', f"Expected bc4e1aa6-cb52-40e9-8f20-594d9c84f920 but got {patient}." + + # 4 hops + research_study = graph.match_label(document_reference_gid, 'ResearchStudy') + assert research_study is not None, "Expected ResearchStudy" + assert research_study['vertex']['gid'] == '7dacd4d0-3c8e-470b-bf61-103891627d45', f"Expected 7dacd4d0-3c8e-470b-bf61-103891627d45 but got {research_study}." + + # Observations + q = graph.V(document_reference_gid).in_(["focus", "subject"]).hasLabel("Observation") + assert len(list(q)) == 2, f"Expected 2 but got {len(list(q))} for {document_reference_gid}." + + +def test_dataframe_associations(graph, expected_vertices, expected_dataframe_associations): + """Test the dataframe associations.""" + + actual_dataframe_associations = defaultdict(list) + # for all objects in the graph except Observations, retrieve the associated objects useful for a dataframe + for vertex_gid, vertex_label in expected_vertices: + if vertex_label == 'Observation': + continue + df = graph.dataframe_associations(vertex_gid, vertex_label) + actual_dataframe_associations[(vertex_label, vertex_gid)] = [(_['resourceType'], _['id']) for _ in df] + assert actual_dataframe_associations == expected_dataframe_associations, f"Expected {expected_dataframe_associations} but got {actual_dataframe_associations}." diff --git a/test/pygrip_test/test_pygrip.py b/test/pygrip_test/test_pygrip.py new file mode 100644 index 000000000..a21f881d2 --- /dev/null +++ b/test/pygrip_test/test_pygrip.py @@ -0,0 +1,29 @@ + +import pygrip +import unittest + +class TestPyGRIP(unittest.TestCase): + + def test_query(self): + + w = pygrip.NewMemServer() + + w.addVertex("1", "Person", {"age":30, "eyes":"brown"}) + w.addVertex("2", "Person", {"age":40, "eyes":"blue"}) + w.addEdge("1", "2", "knows") + + count = 0 + for row in w.V().hasLabel("Person"): + count += 1 + self.assertEqual(count, 2) + + count = 0 + for row in w.V().out("knows"): + count += 1 + self.assertEqual(count, 1) + + for row in w.V().count(): + self.assertEqual(row["count"], 2) + +if __name__ == '__main__': + unittest.main()