Merge pull request #36 from tonindexer/v0.3.3-dev

v0.4

.env.example

@@ -5,4 +5,6 @@ FROM_BLOCK=25000000
 LITESERVERS=135.181.177.59:53312|aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=
 DEBUG_LOGS=false
 WORKERS=4
+RESCAN_WORKERS=4
+RESCAN_SELECT_LIMIT=1000
 # LITESERVERS=65.108.141.177:17439|0MIADpLH4VQn+INHfm0FxGiuZZAA8JfTujRqQugkkA8= # testnet

.golangci.yaml

@@ -55,7 +55,7 @@ linters:
     - whitespace
 linters-settings:
   gocyclo:
-    min-complexity: 16
+    min-complexity: 18
   gosec:
     excludes:
       - G404

Dockerfile

@@ -1,23 +1,30 @@
 # syntax=docker/dockerfile:1.5-labs
-FROM alpine:3 AS emulator-builder
+FROM debian:12.2-slim AS emulator-builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
 
 # build emulator libraries
 
-RUN apk add --no-cache make cmake gcc g++ musl-dev zlib-dev openssl-dev linux-headers git
+RUN apt-get update && \
+    apt-get install -yqq \
+      tzdata build-essential cmake clang openssl \
+      libssl-dev zlib1g-dev gperf wget git curl \
+      libreadline-dev ccache libmicrohttpd-dev ninja-build pkg-config \
+      libsecp256k1-dev libsodium-dev liblz4-dev
 
 ADD --keep-git-dir=true https://github.com/ton-blockchain/ton.git /ton
 RUN cd /ton && git submodule update --init --recursive
 
-RUN apk add --no-cache openblas-dev libmicrohttpd-dev
-
 RUN mkdir build && (cd build && cmake ../ton -DCMAKE_BUILD_TYPE=Release && cmake --build . --target emulator -- -j 8)
 RUN mkdir /output && cp build/emulator/libemulator.so /output
 
 
 # build
-FROM golang:1.19-alpine AS builder
+FROM golang:1.21.4-bookworm AS builder
 
-RUN apk add --no-cache build-base
+RUN apt-get update && \
+    apt-get install -y libsecp256k1-1 libsodium23
 
 #prepare env
 WORKDIR /go/src/github.com/tonindexer/anton
@@ -30,13 +37,14 @@ RUN go mod download
 
 # copy application code
 COPY migrations /go/src/github.com/tonindexer/anton/migrations
+COPY lru /go/src/github.com/tonindexer/anton/lru
 COPY cmd /go/src/github.com/tonindexer/anton/cmd
 COPY addr /go/src/github.com/tonindexer/anton/addr
 COPY abi /go/src/github.com/tonindexer/anton/abi
 COPY internal /go/src/github.com/tonindexer/anton/internal
 COPY main.go /go/src/github.com/tonindexer/anton
 
-RUN rm /go/pkg/mod/github.com/tonkeeper/tongo@v1.1.2/lib/linux/libemulator.so
+RUN rm /go/pkg/mod/github.com/tonkeeper/tongo@v1.3.0/lib/linux/libemulator.so
 COPY --from=emulator-builder /output/libemulator.so /lib/libemulator.so
 
 RUN swag init \
@@ -47,13 +55,15 @@ RUN go build -o /anton /go/src/github.com/tonindexer/anton
 
 
 # application
-FROM alpine:3
+FROM debian:12.2-slim
 
 ENV LISTEN=0.0.0.0:8080
 
-RUN apk add --no-cache libgcc libstdc++
+RUN apt-get update && \
+    apt-get install -y libsecp256k1-1 libsodium23 libssl3 ca-certificates
+
+RUN groupadd anton && useradd -g anton anton
 
-RUN addgroup -S anton && adduser -S anton -G anton
 WORKDIR /app
 COPY --from=builder /lib/libemulator.so /lib
 COPY --from=builder /go/src/github.com/tonindexer/anton/abi/known /var/anton/known

README.md

@@ -57,6 +57,7 @@ To explore how Anton stores data, visit the [migrations' directory](/migrations)
 | `app/parser`      | service determines contract interfaces, parse contract data and message payloads | 
 | `app/fetcher`     | service concurrently fetches data from blockchain                                | 
 | `app/indexer`     | service scans blocks and save parsed data to databases                           |
+| `app/rescan`      | service parses data by updated contract description                              |
 | `app/query`       | service aggregates database repositories                                         |
 | `api/http`        | implements the REST API                                                          |
 
@@ -103,17 +104,19 @@ cp .env.example .env
 nano .env
 ```
 
-| Name                 | Description                       | Default | Example                                                            |
-|----------------------|-----------------------------------|---------|--------------------------------------------------------------------|
-| `DB_NAME`            | Database name                     |         | idx                                                                |
-| `DB_USERNAME`        | Database username                 |         | user                                                               |
-| `DB_PASSWORD`        | Database password                 |         | pass                                                               |
-| `DB_CH_URL`          | Clickhouse URL to connect to      |         | clickhouse://clickhouse:9000/db_name?sslmode=disable               |
-| `DB_PG_URL`          | PostgreSQL URL to connect to      |         | postgres://username:password@postgres:5432/db_name?sslmode=disable |
-| `FROM_BLOCK`         | Master chain seq_no to start from | 1       | 23532000                                                           |
-| `WORKERS`            | Number of indexer workers         | 4       | 8                                                                  |
-| `LITESERVERS`        | Lite servers to connect to        |         | 135.181.177.59:53312 aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=  |
-| `DEBUG_LOGS`         | Debug logs enabled                | false   | true                                                               |
+| Name                  | Description                        | Default | Example                                                            |
+|-----------------------|------------------------------------|---------|--------------------------------------------------------------------|
+| `DB_NAME`             | Database name                      |         | idx                                                                |
+| `DB_USERNAME`         | Database username                  |         | user                                                               |
+| `DB_PASSWORD`         | Database password                  |         | pass                                                               |
+| `DB_CH_URL`           | Clickhouse URL to connect to       |         | clickhouse://clickhouse:9000/db_name?sslmode=disable               |
+| `DB_PG_URL`           | PostgreSQL URL to connect to       |         | postgres://username:password@postgres:5432/db_name?sslmode=disable |
+| `FROM_BLOCK`          | Master chain seq_no to start from  | 1       | 23532000                                                           |
+| `WORKERS`             | Number of indexer workers          | 4       | 8                                                                  |
+| `RESCAN_WORKERS`      | Number of rescan workers           | 4       | 8                                                                  |
+| `RESCAN_SELECT_LIMIT` | Number of rows to fetch for rescan | 3000    | 1000                                                               |
+| `LITESERVERS`         | Lite servers to connect to         |         | 135.181.177.59:53312 aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=  |
+| `DEBUG_LOGS`          | Debug logs enabled                 | false   | true                                                               |
 
 ### Building
 
@@ -157,10 +160,10 @@ To run Anton, you need at least one defined contract interface.
 There are some known interfaces in the [abi/known](/abi/known) directory.
 You can add them through this command:
 ```shell
-docker compose exec web sh -c "anton contract /var/anton/known/*.json"
+docker compose exec rescan sh -c "anton contract addInterfaces /var/anton/known/*.json"
 ```
 
-### Schema migration
+### Database schema migration
 
 ```shell
 # run migrations service on running compose
@@ -209,28 +212,48 @@ docker compose                      \
 
 ## Using
 
-### Show archive nodes from global config
+### Showing archive nodes from global config
 
 ```shell
 docker run tonindexer/anton archive [--testnet]
 ```
 
-### Insert contract interface
+### Inserting contract interface
+
+To add interfaces, you need to provide Anton with a contract description. 
+It will select any interfaces not already present in the database, 
+insert them, and initiate rescan tasks for messages and account states.
 
 ```shell
 # add from stdin
-cat abi/known/tep81_dns.json | docker compose exec -T web anton contract --stdin
+cat abi/known/tep81_dns.json | docker compose exec -T web anton contract addInterfaces --stdin
 # add from file
-docker compose exec web anton contract "/var/anton/known/tep81_dns.json"
+docker compose exec web anton contract addInterfaces "/var/anton/known/tep81_dns.json"
 ```
 
-### Delete contract interface
+### Deleting contract interface
+
+To delete an interface, provide a contract description along with the specific contract name you wish to remove. 
+Anton will then delete the contract interface and its associated operations from the database 
+and initiate rescan tasks to remove all parsed data related to this interface from messages and account states.  
+
+```shell
+docker compose exec rescan sh -c "anton contract deleteInterface -c nft_item /var/anton/known/*.json"
+```
+
+### Updating contract interface
+
+To update a contract interface, you need to provide both the contract description 
+and the specific name of the contract you're updating. 
+Anton will then compare the provided contract interface description against the existing interface in the database. 
+If there are any differences, Anton initiates rescan tasks to reparse data and fix these changes. 
+This process may involve adding, deleting, or updating get-methods and contract operations.
 
 ```shell
-docker compose exec web anton contract delete "dns_nft_item"
+docker compose exec rescan sh -c "anton contract updateInterface -c telemint_nft_item /var/anton/known/telemint.json"
 ```
 
-### Add address label
+### Adding address label
 
 ```shell
 docker compose exec web anton label "EQDj5AA8mQvM5wJEQsFFFof79y3ZsuX6wowktWQFhz_Anton" "anton.tools"