Use ast files from packages.Load, remove parser.ParseFile, some minor…

… refactoring

MAINTAINERS

@@ -1,3 +1,4 @@
 Sergey Bykov  <[email protected]>
 Vladimir Tereschenko <[email protected]>
-Andrei Simonov <[email protected]>
+Andrei Simonov <[email protected]>
+Mikhail Eremin <[email protected]>

go.sum

@@ -9,6 +9,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -73,6 +74,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/semrush/zenrpc v1.1.1 h1:McE4BFoXP95NnDU+tQHhfzVpmODS4p55JKXxHR64nx4=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=

parser/helpers.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"fmt"
 	"github.com/thoas/go-funk"
+	"go/ast"
 	"golang.org/x/tools/go/packages"
 	"path"
 	"strings"
@@ -40,6 +41,30 @@ func getDependenciesFilenames(dir string) ([]string, error) {
 	return funk.UniqString(goFiles), nil
 }
 
+func GetDependenciesAstFiles(filename string) ([]*ast.File, error) {
+	pkgs, err := loadPackageWithSyntax(path.Dir(filename))
+	if err != nil {
+		return nil, err
+	}
+	astFiles := []*ast.File{}
+	done := map[string]bool{}
+	for _, pkg := range pkgs {
+		if _, ok := done[pkg.Name]; ok {
+			continue
+		}
+		astFiles = append(astFiles, pkg.Syntax...)
+		done[pkg.Name] = true
+		for _, childPack := range pkg.Imports {
+			if _, ok := done[childPack.Name]; ok {
+				continue
+			}
+			astFiles = append(astFiles, childPack.Syntax...)
+			done[childPack.Name] = true
+		}
+	}
+	return astFiles, nil
+}
+
 func goFilesFromPackage(pkg *packages.Package) []string {
 	files := []string{}
 	files = append(files, pkg.GoFiles...)
@@ -62,3 +87,12 @@ func loadPackage(path string) ([]*packages.Package, error) {
 		Mode: packages.NeedImports | packages.NeedFiles | packages.NeedName,
 	}, path)
 }
+
+func loadPackageWithSyntax(path string) ([]*packages.Package, error) {
+	return packages.Load(&packages.Config{
+		Mode: packages.NeedImports |
+			packages.NeedFiles |
+			packages.NeedName |
+			packages.NeedSyntax,
+	}, path)
+}

parser/helpers_test.go

@@ -2,14 +2,12 @@ package parser
 
 import (
 	. "github.com/smartystreets/goconvey/convey"
-	"github.com/thoas/go-funk"
 	"testing"
 )
 
-func TestLoadPackagesRecursive(t *testing.T) {
-	Convey("Should get files from entrypoint", t, func() {
-		files, err := GetDependencies("github.com/semrush/zenrpc/v2/testdata/subservice/subarithservice.go")
+func TestLoadPackage(t *testing.T) {
+	Convey("Should load package with syntax and imports", t, func() {
+		_, err := loadPackage("../testdata/subservice/subarithservice.go")
 		So(err, ShouldBeNil)
-		So(files, ShouldHaveLength, len(funk.UniqString(files)))
 	})
 }

parser/parser.go

@@ -3,10 +3,7 @@ package parser
 import (
 	"fmt"
 	"go/ast"
-	"go/parser"
 	"go/token"
-	"io/ioutil"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -28,10 +25,11 @@ const (
 
 // PackageInfo represents struct info for XXX_zenrpc.go file generation
 type PackageInfo struct {
-	Dir string
-
+	EntryPoint  string
+	Dir         string
 	PackageName string
-	Services    []*Service
+
+	Services []*Service
 
 	Scopes  map[string][]*ast.Scope // key - import name, value - array of scopes from each package file
 	Structs map[string]*Struct
@@ -119,41 +117,53 @@ type SMDError struct {
 	Description string
 }
 
-func NewPackageInfo() *PackageInfo {
+func NewPackageInfo(filename string) (*PackageInfo, error) {
+	dir, err := filepath.Abs(filepath.Dir(filename))
+	if err != nil {
+		return nil, err
+	}
+
+	packageName, err := EntryPointPackageName(filename)
+	if err != nil {
+		return nil, err
+	}
+
 	return &PackageInfo{
-		Services: []*Service{},
+		EntryPoint:  filename,
+		Dir:         dir,
+		PackageName: packageName,
+		Services:    []*Service{},
 
 		Scopes:  make(map[string][]*ast.Scope),
 		Structs: make(map[string]*Struct),
 		Imports: []*ast.ImportSpec{},
 
 		StructsNamespacesFromArgs: make(map[string]struct{}),
 		ImportsForGeneration:      []*ast.ImportSpec{},
-	}
+	}, nil
 }
 
 // ParseFiles parse all files associated with package from original file
 func (pi *PackageInfo) Parse(filename string) error {
-	if dir, err := filepath.Abs(filepath.Dir(filename)); err != nil {
-		return err
-	} else {
-		pi.Dir = dir
-	}
-
-	err := pi.rememberEntryPointPackageName(filename)
+	astFiles, err := GetDependenciesAstFiles(filename)
 	if err != nil {
 		return err
 	}
 
-	// filePaths, err := pi.scanDirectoryFilePathsRecursive(pi.Dir, []string{})
-	// use recursive dependencies getter from helpers
-	filePaths, err := GetDependencies(filename)
-	if err != nil {
-		return err
+	for _, astFile := range astFiles {
+		// collect scopes
+		pi.collectScopes(astFile)
+		// get structs for zenrpc
+		pi.collectServices(astFile)
+		// get imports
+		pi.collectImports(astFile)
 	}
 
-	if err := pi.parseFiles(filePaths); err != nil {
-		return err
+	// second loop: parse methods
+	for _, f := range astFiles {
+		if err := pi.parseMethods(f); err != nil {
+			return err
+		}
 	}
 
 	// collect scopes from imported packages
@@ -169,92 +179,19 @@ func (pi *PackageInfo) Parse(filename string) error {
 	return nil
 }
 
-func (pi *PackageInfo) parseOsFileToAstFile(filename string) (*ast.File, error) {
-	astFile, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.ParseComments)
-	if err != nil {
-		return nil, err
-	}
-
-	// for debug
-	//ast.Print(fset, astFile)
-
+func (pi *PackageInfo) collectScopes(astFile *ast.File) {
 	if pi.PackageName != astFile.Name.Name {
 		pi.Scopes[astFile.Name.Name] = append(pi.Scopes[astFile.Name.Name], astFile.Scope) // collect other package scopes
 	} else {
 		pi.Scopes["."] = append(pi.Scopes["."], astFile.Scope) // collect current package scopes
 	}
-
-	// get structs for zenrpc
-	pi.parseServices(astFile)
-
-	pi.Imports = append(pi.Imports, astFile.Imports...) // collect imports
-	return astFile, nil
 }
 
-func (pi *PackageInfo) scanDirectoryFilePathsRecursive(dirPath string, paths []string) ([]string, error) {
-	files, err := ioutil.ReadDir(dirPath)
-	if err != nil {
-		return nil, err
-	}
-	for _, file := range files {
-		if !file.IsDir() {
-			paths = append(paths, path.Join(dirPath, file.Name()))
-		} else {
-			paths, err = pi.scanDirectoryFilePathsRecursive(path.Join(dirPath, file.Name()), paths)
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	return paths, nil
-}
-
-func (pi *PackageInfo) rememberEntryPointPackageName(entrypoint string) error {
-	// remember current package name from first incoming file
-	if len(pi.PackageName) == 0 {
-		packageName, err := EntryPointPackageName(entrypoint)
-		if err != nil {
-			return err
-		}
-		pi.PackageName = packageName
-	}
-	return nil
-}
-func (pi *PackageInfo) astFiles(filepaths []string) ([]*ast.File, error) {
-	astFiles := []*ast.File{}
-	// first loop: parse suitable files and structs
-	for _, f := range filepaths {
-		if pi.isSuitableFile(f) {
-			astFile, err := pi.parseOsFileToAstFile(f)
-			if err != nil {
-				return nil, err
-			}
-			astFiles = append(astFiles, astFile)
-		}
-	}
-
-	return astFiles, nil
-}
-
-func (pi *PackageInfo) parseFiles(filepaths []string) error {
-	// first loop, get needed ast files and their structs
-
-	astFiles, err := pi.astFiles(filepaths)
-	if err != nil {
-		return err
-	}
-
-	// second loop: parse methods
-	for _, f := range astFiles {
-		if err := pi.parseMethods(f); err != nil {
-			return err
-		}
-	}
-
-	return nil
+func (pi *PackageInfo) collectImports(astFile *ast.File) {
+	pi.Imports = append(pi.Imports, astFile.Imports...) // collect imports
 }
 
-func (pi *PackageInfo) parseServices(f *ast.File) {
+func (pi *PackageInfo) collectServices(f *ast.File) {
 	for _, decl := range f.Decls {
 		gdecl, ok := decl.(*ast.GenDecl)
 		if !ok || gdecl.Tok != token.TYPE {
@@ -289,14 +226,6 @@ func (pi *PackageInfo) parseServices(f *ast.File) {
 	}
 }
 
-func (pi *PackageInfo) isSuitableFile(filepath string) bool {
-	if !strings.HasSuffix(filepath, goFileSuffix) ||
-		strings.HasSuffix(filepath, GenerateFileSuffix) || strings.HasSuffix(filepath, testFileSuffix) {
-		return false
-	}
-	return true
-}
-
 func (pi *PackageInfo) parseMethods(f *ast.File) error {
 	for _, decl := range f.Decls {
 		fdecl, ok := decl.(*ast.FuncDecl)
@@ -387,6 +316,10 @@ func (pi PackageInfo) String() string {
 	return result
 }
 
+func (pi PackageInfo) OutputFilename() string {
+	return filepath.Join(pi.Dir, pi.PackageName+GenerateFileSuffix)
+}
+
 // HasErrorVariable define adding err variable to generated Invoke function
 func (s Service) HasErrorVariable() bool {
 	for _, m := range s.Methods {

testdata/model/model.go

@@ -1,6 +1,9 @@
 package model
 
+import "github.com/semrush/zenrpc/v2/testdata/objects"
+
 type Point struct {
+	objects.AbstractObject
 	X, Y int // coordinate
 	Z    int `json:"-"`
 }

testdata/subservice/subarithservice.go

@@ -21,7 +21,7 @@ func (as SubArithService) Positive() (bool, *zenrpc.Error) {
 	return true, nil
 }
 
-func (SubArithService) ReturnPointFromSamePackage() Point {
+func (SubArithService) ReturnPointFromSamePackage(p Point) Point {
 	// some optimistic operations
 	return Point{}
 }