From ee55e96465cec1fddcf56cb41e1c8232d380ffc6 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:12:40 -0500 Subject: [PATCH 1/2] initialize token transfers --- notebooks/adhoc/token_transfers_dev.ipynb | 257 ++++++++++++++++++ .../models/code/token_transfers.py | 41 +++ .../models/templates/native_transfers.sql.j2 | 36 +++ 3 files changed, 334 insertions(+) create mode 100644 notebooks/adhoc/token_transfers_dev.ipynb create mode 100644 src/op_analytics/datapipeline/models/code/token_transfers.py create mode 100644 src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 diff --git a/notebooks/adhoc/token_transfers_dev.ipynb b/notebooks/adhoc/token_transfers_dev.ipynb new file mode 100644 index 00000000000..bb0830977c9 --- /dev/null +++ b/notebooks/adhoc/token_transfers_dev.ipynb @@ -0,0 +1,257 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare data reader for a given chain and date" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1mdebug \u001b[0m] \u001b[1mconnecting to OPLABS Clickhouse client...\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m25\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mloaded vault from .env file \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mvault.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m32\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1mdebug \u001b[0m] \u001b[1mloaded vault: 17 items \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mvault.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m76\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1mdebug \u001b[0m] \u001b[1minitialized OPLABS Clickhouse client.\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m37\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mprepared 1 input batches. \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mbydate.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m95\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n" + ] + } + ], + "source": [ + "from op_analytics.coreutils.duckdb_inmem import init_client\n", + "from op_analytics.coreutils.partitioned.reader import DataReader\n", + "from op_analytics.coreutils.partitioned.location import DataLocation\n", + "from op_analytics.datapipeline.etl.intermediate.construct import construct_data_readers\n", + "\n", + "from op_analytics.datapipeline.models.compute.udfs import create_duckdb_macros\n", + "\n", + "\n", + "# Define the input data range.\n", + "read_batches: list[DataReader] = construct_data_readers(\n", + " chains=[\"op\"],\n", + " models=[\"token_transfers\"],\n", + " range_spec=\"@20241030:+1\",\n", + " read_from=DataLocation.GCS\n", + ")\n", + "\n", + "\n", + "# Select input for one date and build the intermediate model inputs.\n", + "batch = read_batches[0]\n", + "\n", + "\n", + "duckdb_client = init_client()\n", + "create_duckdb_macros(duckdb_client)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run the model\n", + "\n", + "This automatically registers the model outputs as duckdb tables." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mExecuting model function... \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mtestutils.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m220\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mduckdb dataset='ingestion/traces_v1' using 23/23 parquet paths, first path is gs://oplabs-tools-data-sink/ingestion/traces_v1/chain=op/dt=2024-10-30/000127322000.parquet\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mreader.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:23\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mregistered view: 'ingestion_traces_v1' using 23 parquet paths\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m53\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:23\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mduckdb dataset='ingestion/logs_v1' using 23/23 parquet paths, first path is gs://oplabs-tools-data-sink/ingestion/logs_v1/chain=op/dt=2024-10-30/000127322000.parquet\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mreader.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:24\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mregistered view: 'ingestion_logs_v1' using 23 parquet paths\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m53\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:24\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mduckdb dataset='ingestion/transactions_v1' using 23/23 parquet paths, first path is gs://oplabs-tools-data-sink/ingestion/transactions_v1/chain=op/dt=2024-10-30/000127322000.parquet\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mreader.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:26\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mregistered view: 'ingestion_transactions_v1' using 23 parquet paths\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m53\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", + "\u001b[2m2024-12-17 18:00:26\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mRendering query \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mquerybuilder.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m40\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m \u001b[36mtemplate\u001b[0m=\u001b[35mnative_transfers\u001b[0m\n", + "dict_keys(['native_transfers_v1'])\n" + ] + }, + { + "data": { + "text/plain": [ + "┌───────────────────────────┐\n", + "│ name │\n", + "│ varchar │\n", + "├───────────────────────────┤\n", + "│ ingestion_logs_v1 │\n", + "│ ingestion_traces_v1 │\n", + "│ ingestion_transactions_v1 │\n", + "│ native_transfers │\n", + "│ native_transfers_v1 │\n", + "└───────────────────────────┘" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from op_analytics.datapipeline.models.compute.testutils import execute_model_in_memory\n", + "\n", + "\n", + "execute_model_in_memory(\n", + " duckdb_client=duckdb_client,\n", + " model=\"token_transfers\",\n", + " data_reader=batch,\n", + ")\n", + "\n", + "# The duckdb database will have the following:\n", + "# - input tables\n", + "# - views used by the model\n", + "# - model outputs\n", + "#\n", + "# You can use duckdb to inspect any of the above results.\n", + "duckdb_client.sql(\"SHOW TABLES\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify model results" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "┌─────────┬──────────┬─────────┬────────────┬─────────────────┬──────────────┬────────────────────────────────────────────────────────────────────┬────────────────────────────────────────────────────────────────────┬───────────────────┬────────────────────────────────────────────┬────────────────────────────────────────────┬─────────────────┬────────────────────┬────────────────────┬────────────┬───────────┬─────────────────┬────────────────┬───────────┬───────────────┬────────────────────────────────────────────┬────────────────────────────────────────────┬──────────────┬──────────────────────┐\n", + "│ network │ chain_id │ chain │ dt │ block_timestamp │ block_number │ block_hash │ transaction_hash │ transaction_index │ transfer_from_address │ transfer_to_address │ trace_method_id │ value_64 │ value_lossless │ trace_type │ call_type │ trace_gas_limit │ trace_gas_used │ subtraces │ trace_address │ tx_from_address │ tx_to_address │ tx_method_id │ value_native │\n", + "│ varchar │ int32 │ varchar │ date │ uint32 │ int64 │ varchar │ varchar │ int64 │ varchar │ varchar │ varchar │ int64 │ varchar │ varchar │ varchar │ int64 │ int64 │ int64 │ varchar │ varchar │ varchar │ varchar │ double │\n", + "├─────────┼──────────┼─────────┼────────────┼─────────────────┼──────────────┼────────────────────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼───────────────────┼────────────────────────────────────────────┼────────────────────────────────────────────┼─────────────────┼────────────────────┼────────────────────┼────────────┼───────────┼─────────────────┼────────────────┼───────────┼───────────────┼────────────────────────────────────────────┼────────────────────────────────────────────┼──────────────┼──────────────────────┤\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250601 │ 127325912 │ 0x091e27e20a77c42779efb7c7af674917a25ed835bb0e87fbc062993d109f4608 │ 0xaf039d9965ee1ed1a047d06fe80f119a449476d59f44bfbda342757147be6140 │ 7 │ 0x1632ed73c1a866fe2c21e6fd0f49687f445e2adb │ 0xe4edb277e41dc89ab076a1f049f4a3efa700bce8 │ 0x │ 269375236255299057 │ 269375236255299057 │ call │ call │ 21000 │ 21000 │ 0 │ │ 0x1632ed73c1a866fe2c21e6fd0f49687f445e2adb │ 0xe4edb277e41dc89ab076a1f049f4a3efa700bce8 │ 0x │ 0.2693752362552991 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xf958157473bb2d066d8225ec07ecfef26415c095812d0757165fd6606d5fb206 │ 34 │ 0x4200000000000000000000000000000000000006 │ 0x1c6858664ccfacc2dcc8175f663cfb13ef13bb57 │ 0x │ 57459274561 │ 57459274561 │ call │ call │ 2300 │ 75 │ 0 │ 26,6,3,0 │ 0x3c9779d3ec32ed4985a3e634c44d38c517564f5b │ 0x346c0b53faea3eaa013aebe3019b0bf09448aeb1 │ 0x00001a39 │ 5.7459274561e-08 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xf958157473bb2d066d8225ec07ecfef26415c095812d0757165fd6606d5fb206 │ 34 │ 0x1c6858664ccfacc2dcc8175f663cfb13ef13bb57 │ 0xabe58b50367fb4cea7d19c41c228abed381ed2ef │ 0x │ 57459274561 │ 57459274561 │ call │ call │ 2300 │ 0 │ 0 │ 26,6,4 │ 0x3c9779d3ec32ed4985a3e634c44d38c517564f5b │ 0x346c0b53faea3eaa013aebe3019b0bf09448aeb1 │ 0x00001a39 │ 5.7459274561e-08 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x995ff15d8712d4b727f388dd9814841b9063801bd53fe3baccea3bf0b5a1c9fa │ 40 │ 0xc7b7bccba82df66e9c219751fcf300d75fab5c05 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0xba677db7 │ 2418622215047 │ 2418622215047 │ call │ call │ 486358 │ 433983 │ 13 │ │ 0xc7b7bccba82df66e9c219751fcf300d75fab5c05 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0xba677db7 │ 2.418622215047e-06 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x995ff15d8712d4b727f388dd9814841b9063801bd53fe3baccea3bf0b5a1c9fa │ 40 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0x94a365ca808029af8db18257ecd296c16c61ac05 │ 0x │ 2418622215047 │ 2418622215047 │ call │ call │ 377551 │ 0 │ 0 │ 3 │ 0xc7b7bccba82df66e9c219751fcf300d75fab5c05 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0xba677db7 │ 2.418622215047e-06 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xe53957d3c4e3e819bbeb522bd1cf097f3c30abfe65f2a51303534ca00e141757 │ 39 │ 0x5d7e4ca0cd3d9aa42724a61d139cd93cbbadf825 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0xfc180638 │ 1557961691647504 │ 1557961691647504 │ call │ call │ 38283 │ 37917 │ 1 │ │ 0x5d7e4ca0cd3d9aa42724a61d139cd93cbbadf825 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0xfc180638 │ 0.001557961691647504 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xe53957d3c4e3e819bbeb522bd1cf097f3c30abfe65f2a51303534ca00e141757 │ 39 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0x5e809a85aa182a9921edd10a4163745bb3e36284 │ 0x │ 1557961691647504 │ 1557961691647504 │ call │ call │ 4897 │ 0 │ 0 │ 0 │ 0x5d7e4ca0cd3d9aa42724a61d139cd93cbbadf825 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0xfc180638 │ 0.001557961691647504 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x8adc1668c8b8a55c6866e791eb70c343e33d89043c79b3bc19e9f74746596dfb │ 22 │ 0xf32314be92973a6d7c7ec60cdf114fa331e8f8e5 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x29723511 │ 380008468139144441 │ 380008468139144441 │ call │ call │ 58738 │ 45132 │ 1 │ │ 0xf32314be92973a6d7c7ec60cdf114fa331e8f8e5 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x29723511 │ 0.38000846813914446 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x8adc1668c8b8a55c6866e791eb70c343e33d89043c79b3bc19e9f74746596dfb │ 22 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x80c67432656d59144ceff962e8faf8926599bcf8 │ 0x │ 380008468139144441 │ 380008468139144441 │ call │ call │ 2300 │ 0 │ 0 │ 0 │ 0xf32314be92973a6d7c7ec60cdf114fa331e8f8e5 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x29723511 │ 0.38000846813914446 │\n", + "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xc1cddca85df209a6c9eaca445001a2592f69c8a53d92618832ab42b095765f78 │ 23 │ 0x0000000071727de22e5e9d8baf0edac6f37da032 │ 0x433702873e33d4846d399a75aaf96eacf181d0b2 │ 0x │ 1140653506944 │ 1140653506944 │ call │ call │ 992723 │ 0 │ 0 │ 3 │ 0x433702873e33d4846d399a75aaf96eacf181d0b2 │ 0x0000000071727de22e5e9d8baf0edac6f37da032 │ 0x765e827f │ 1.140653506944e-06 │\n", + "├─────────┴──────────┴─────────┴────────────┴─────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────┴───────────────────┴────────────────────────────────────────────┴────────────────────────────────────────────┴─────────────────┴────────────────────┴────────────────────┴────────────┴───────────┴─────────────────┴────────────────┴───────────┴───────────────┴────────────────────────────────────────────┴────────────────────────────────────────────┴──────────────┴──────────────────────┤\n", + "│ 10 rows 24 columns │\n", + "└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "duckdb_client.sql(\"SELECT *, value_64/1e18 AS value_native FROM native_transfers_v1 LIMIT 10\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### You can also convert the results to dataframes to inspect them in more familiar ways" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (5, 23)
networkchain_idchaindtblock_timestampblock_numberblock_hashtransaction_hashtransaction_indextransfer_from_addresstransfer_to_addresstrace_method_idvalue_64value_losslesstrace_typecall_typetrace_gas_limittrace_gas_usedsubtracestrace_addresstx_from_addresstx_to_addresstx_method_id
stri32strdateu32i64strstri64strstrstri64strstrstri64i64i64strstrstrstr
"mainnet"10"op"2024-10-301730250601127325912"0x091e27e20a77c42779efb7c7af67…"0xaf039d9965ee1ed1a047d06fe80f…7"0x1632ed73c1a866fe2c21e6fd0f49…"0xe4edb277e41dc89ab076a1f049f4…"0x"269375236255299057"269375236255299057""call""call"21000210000"""0x1632ed73c1a866fe2c21e6fd0f49…"0xe4edb277e41dc89ab076a1f049f4…"0x"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0xf958157473bb2d066d8225ec07ec…34"0x4200000000000000000000000000…"0x1c6858664ccfacc2dcc8175f663c…"0x"57459274561"57459274561""call""call"2300750"26,6,3,0""0x3c9779d3ec32ed4985a3e634c44d…"0x346c0b53faea3eaa013aebe3019b…"0x00001a39"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0xf958157473bb2d066d8225ec07ec…34"0x1c6858664ccfacc2dcc8175f663c…"0xabe58b50367fb4cea7d19c41c228…"0x"57459274561"57459274561""call""call"230000"26,6,4""0x3c9779d3ec32ed4985a3e634c44d…"0x346c0b53faea3eaa013aebe3019b…"0x00001a39"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0x995ff15d8712d4b727f388dd9814…40"0xc7b7bccba82df66e9c219751fcf3…"0xa2a786ff9148f7c88ee93372db8c…"0xba677db7"2418622215047"2418622215047""call""call"48635843398313"""0xc7b7bccba82df66e9c219751fcf3…"0xa2a786ff9148f7c88ee93372db8c…"0xba677db7"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0x995ff15d8712d4b727f388dd9814…40"0xa2a786ff9148f7c88ee93372db8c…"0x94a365ca808029af8db18257ecd2…"0x"2418622215047"2418622215047""call""call"37755100"3""0xc7b7bccba82df66e9c219751fcf3…"0xa2a786ff9148f7c88ee93372db8c…"0xba677db7"
" + ], + "text/plain": [ + "shape: (5, 23)\n", + "┌─────────┬──────────┬───────┬────────────┬───┬─────────────┬────────────┬────────────┬────────────┐\n", + "│ network ┆ chain_id ┆ chain ┆ dt ┆ … ┆ trace_addre ┆ tx_from_ad ┆ tx_to_addr ┆ tx_method_ │\n", + "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ ss ┆ dress ┆ ess ┆ id │\n", + "│ str ┆ i32 ┆ str ┆ date ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", + "│ ┆ ┆ ┆ ┆ ┆ str ┆ str ┆ str ┆ str │\n", + "╞═════════╪══════════╪═══════╪════════════╪═══╪═════════════╪════════════╪════════════╪════════════╡\n", + "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ ┆ 0x1632ed73 ┆ 0xe4edb277 ┆ 0x │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ c1a866fe2c ┆ e41dc89ab0 ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 21e6fd0f49 ┆ 76a1f049f4 ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", + "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ 26,6,3,0 ┆ 0x3c9779d3 ┆ 0x346c0b53 ┆ 0x00001a39 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ec32ed4985 ┆ faea3eaa01 ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ a3e634c44d ┆ 3aebe3019b ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", + "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ 26,6,4 ┆ 0x3c9779d3 ┆ 0x346c0b53 ┆ 0x00001a39 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ec32ed4985 ┆ faea3eaa01 ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ a3e634c44d ┆ 3aebe3019b ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", + "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ ┆ 0xc7b7bccb ┆ 0xa2a786ff ┆ 0xba677db7 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ a82df66e9c ┆ 9148f7c88e ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 219751fcf3 ┆ e93372db8c ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", + "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ 3 ┆ 0xc7b7bccb ┆ 0xa2a786ff ┆ 0xba677db7 │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ a82df66e9c ┆ 9148f7c88e ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ 219751fcf3 ┆ e93372db8c ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", + "└─────────┴──────────┴───────┴────────────┴───┴─────────────┴────────────┴────────────┴────────────┘" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "duckdb_client.sql(\"SELECT * FROM native_transfers_v1 LIMIT 10\").pl().head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/op_analytics/datapipeline/models/code/token_transfers.py b/src/op_analytics/datapipeline/models/code/token_transfers.py new file mode 100644 index 00000000000..5ede37295bd --- /dev/null +++ b/src/op_analytics/datapipeline/models/code/token_transfers.py @@ -0,0 +1,41 @@ +import duckdb + +from op_analytics.datapipeline.models.compute.querybuilder import TemplatedSQLQuery +from op_analytics.datapipeline.models.compute.registry import register_model +from op_analytics.datapipeline.models.compute.types import NamedRelations + + +@register_model( + input_datasets=["ingestion/traces_v1", "ingestion/logs_v1", "ingestion/transactions_v1"], + expected_outputs=["native_transfers_v1"], + auxiliary_views=[ + TemplatedSQLQuery( + template_name="native_transfers", + context={}, + ), + # TemplatedSQLQuery( + # template_name="erc20_transfers", + # context={}, + # ), + ], +) +def token_transfers(duckdb_client: duckdb.DuckDBPyConnection) -> NamedRelations: + # Generic Function to decode events with the ABI - Will Prototype in dev notebook? + # def decode_logs(df): + # Get Unique Topic0s + # Grab ABIs for Topic0s - From some log we have? Maybe look up to a repo eventually? + # Decode the appropriate row based on Topic0 <> ABI Mapping + # Retrun Decoded Table + # return + # Get raw filtered logs for each token standard + # erc20_view = duckdb_client.view("erc20_transfer_raw_logs") + # erc721_view = duckdb_client.view("erc721_transfer_raw_logs") + # erc1155_view = duckdb_client.view("erc1155_transfer_raw_logs") + + # decoded_erc20_view = decode_events(erc20_view, abi) + + return { + # TBD if we materialize these individually, or roll them up in to a token transfers table + "native_transfers_v1": duckdb_client.view("native_transfers"), + # "erc20_transfers_v1": decoded_erc20_view, + } diff --git a/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 b/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 new file mode 100644 index 00000000000..1da74eef51d --- /dev/null +++ b/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 @@ -0,0 +1,36 @@ +SELECT + tr.network + , tr.chain_id + , tr.chain + , tr.dt + , tr.block_timestamp + , tr.block_number + , tr.block_hash + , tr.transaction_hash + , tr.transaction_index + , tr.from_address AS transfer_from_address + , tr.to_address AS transfer_to_address + , hexstr_method_id(tr.input) AS trace_method_id + , tr.value_64 + , tr.value_lossless + , tr.trace_type + , tr.call_type + , tr.gas AS trace_gas_limit + , tr.gas_used AS trace_gas_used + , tr.subtraces + , tr.trace_address + , t.from_address AS tx_from_address + , t.to_address AS tx_to_address + , hexstr_method_id(t.input) AS tx_method_id + +FROM ingestion_traces_v1 AS tr +INNER JOIN ingestion_transactions_v1 AS t + ON + tr.transaction_hash = t.hash + AND tr.block_number = t.block_number + AND tr.chain_id = t.chain_id +WHERE + (tr.call_type NOT IN ('delegatecall', 'callcode', 'staticcall') OR tr.call_type = '') + AND t.receipt_status = 1 + AND tr.status = 1 + AND tr.value_lossless != '0' From da1b919d47c224b9baf6320460ca2a020c9c9b84 Mon Sep 17 00:00:00 2001 From: Michael Silberling <4006780+MSilb7@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:24:31 -0500 Subject: [PATCH 2/2] test --- notebooks/adhoc/token_transfers_dev.ipynb | 150 ++---------------- .../models/code/token_transfers.py | 63 +++++--- .../datapipeline/models/compute/udfs.py | 2 +- .../filtered_raw_erc20_transfer_logs.sql.j2 | 16 ++ .../models/templates/filtered_raw_logs.sql.j2 | 14 ++ .../models/templates/native_transfers.sql.j2 | 1 - 6 files changed, 85 insertions(+), 161 deletions(-) create mode 100644 src/op_analytics/datapipeline/models/templates/filtered_raw_erc20_transfer_logs.sql.j2 create mode 100644 src/op_analytics/datapipeline/models/templates/filtered_raw_logs.sql.j2 diff --git a/notebooks/adhoc/token_transfers_dev.ipynb b/notebooks/adhoc/token_transfers_dev.ipynb index bb0830977c9..4ed7bcfda6a 100644 --- a/notebooks/adhoc/token_transfers_dev.ipynb +++ b/notebooks/adhoc/token_transfers_dev.ipynb @@ -9,21 +9,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1mdebug \u001b[0m] \u001b[1mconnecting to OPLABS Clickhouse client...\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m25\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mloaded vault from .env file \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mvault.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m32\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1mdebug \u001b[0m] \u001b[1mloaded vault: 17 items \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mvault.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m76\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1mdebug \u001b[0m] \u001b[1minitialized OPLABS Clickhouse client.\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m37\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mprepared 1 input batches. \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mbydate.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m95\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "from op_analytics.coreutils.duckdb_inmem import init_client\n", "from op_analytics.coreutils.partitioned.reader import DataReader\n", @@ -61,44 +49,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mExecuting model function... \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mtestutils.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m220\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:22\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mduckdb dataset='ingestion/traces_v1' using 23/23 parquet paths, first path is gs://oplabs-tools-data-sink/ingestion/traces_v1/chain=op/dt=2024-10-30/000127322000.parquet\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mreader.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:23\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mregistered view: 'ingestion_traces_v1' using 23 parquet paths\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m53\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:23\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mduckdb dataset='ingestion/logs_v1' using 23/23 parquet paths, first path is gs://oplabs-tools-data-sink/ingestion/logs_v1/chain=op/dt=2024-10-30/000127322000.parquet\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mreader.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:24\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mregistered view: 'ingestion_logs_v1' using 23 parquet paths\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m53\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:24\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mduckdb dataset='ingestion/transactions_v1' using 23/23 parquet paths, first path is gs://oplabs-tools-data-sink/ingestion/transactions_v1/chain=op/dt=2024-10-30/000127322000.parquet\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mreader.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m72\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:26\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mregistered view: 'ingestion_transactions_v1' using 23 parquet paths\u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mclient.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m53\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m\n", - "\u001b[2m2024-12-17 18:00:26\u001b[0m [\u001b[32m\u001b[1minfo \u001b[0m] \u001b[1mRendering query \u001b[0m \u001b[36mfilename\u001b[0m=\u001b[35mquerybuilder.py\u001b[0m \u001b[36mlineno\u001b[0m=\u001b[35m40\u001b[0m \u001b[36mprocess\u001b[0m=\u001b[35m5921\u001b[0m \u001b[36mtemplate\u001b[0m=\u001b[35mnative_transfers\u001b[0m\n", - "dict_keys(['native_transfers_v1'])\n" - ] - }, - { - "data": { - "text/plain": [ - "┌───────────────────────────┐\n", - "│ name │\n", - "│ varchar │\n", - "├───────────────────────────┤\n", - "│ ingestion_logs_v1 │\n", - "│ ingestion_traces_v1 │\n", - "│ ingestion_transactions_v1 │\n", - "│ native_transfers │\n", - "│ native_transfers_v1 │\n", - "└───────────────────────────┘" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from op_analytics.datapipeline.models.compute.testutils import execute_model_in_memory\n", "\n", @@ -127,102 +80,27 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "┌─────────┬──────────┬─────────┬────────────┬─────────────────┬──────────────┬────────────────────────────────────────────────────────────────────┬────────────────────────────────────────────────────────────────────┬───────────────────┬────────────────────────────────────────────┬────────────────────────────────────────────┬─────────────────┬────────────────────┬────────────────────┬────────────┬───────────┬─────────────────┬────────────────┬───────────┬───────────────┬────────────────────────────────────────────┬────────────────────────────────────────────┬──────────────┬──────────────────────┐\n", - "│ network │ chain_id │ chain │ dt │ block_timestamp │ block_number │ block_hash │ transaction_hash │ transaction_index │ transfer_from_address │ transfer_to_address │ trace_method_id │ value_64 │ value_lossless │ trace_type │ call_type │ trace_gas_limit │ trace_gas_used │ subtraces │ trace_address │ tx_from_address │ tx_to_address │ tx_method_id │ value_native │\n", - "│ varchar │ int32 │ varchar │ date │ uint32 │ int64 │ varchar │ varchar │ int64 │ varchar │ varchar │ varchar │ int64 │ varchar │ varchar │ varchar │ int64 │ int64 │ int64 │ varchar │ varchar │ varchar │ varchar │ double │\n", - "├─────────┼──────────┼─────────┼────────────┼─────────────────┼──────────────┼────────────────────────────────────────────────────────────────────┼────────────────────────────────────────────────────────────────────┼───────────────────┼────────────────────────────────────────────┼────────────────────────────────────────────┼─────────────────┼────────────────────┼────────────────────┼────────────┼───────────┼─────────────────┼────────────────┼───────────┼───────────────┼────────────────────────────────────────────┼────────────────────────────────────────────┼──────────────┼──────────────────────┤\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250601 │ 127325912 │ 0x091e27e20a77c42779efb7c7af674917a25ed835bb0e87fbc062993d109f4608 │ 0xaf039d9965ee1ed1a047d06fe80f119a449476d59f44bfbda342757147be6140 │ 7 │ 0x1632ed73c1a866fe2c21e6fd0f49687f445e2adb │ 0xe4edb277e41dc89ab076a1f049f4a3efa700bce8 │ 0x │ 269375236255299057 │ 269375236255299057 │ call │ call │ 21000 │ 21000 │ 0 │ │ 0x1632ed73c1a866fe2c21e6fd0f49687f445e2adb │ 0xe4edb277e41dc89ab076a1f049f4a3efa700bce8 │ 0x │ 0.2693752362552991 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xf958157473bb2d066d8225ec07ecfef26415c095812d0757165fd6606d5fb206 │ 34 │ 0x4200000000000000000000000000000000000006 │ 0x1c6858664ccfacc2dcc8175f663cfb13ef13bb57 │ 0x │ 57459274561 │ 57459274561 │ call │ call │ 2300 │ 75 │ 0 │ 26,6,3,0 │ 0x3c9779d3ec32ed4985a3e634c44d38c517564f5b │ 0x346c0b53faea3eaa013aebe3019b0bf09448aeb1 │ 0x00001a39 │ 5.7459274561e-08 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xf958157473bb2d066d8225ec07ecfef26415c095812d0757165fd6606d5fb206 │ 34 │ 0x1c6858664ccfacc2dcc8175f663cfb13ef13bb57 │ 0xabe58b50367fb4cea7d19c41c228abed381ed2ef │ 0x │ 57459274561 │ 57459274561 │ call │ call │ 2300 │ 0 │ 0 │ 26,6,4 │ 0x3c9779d3ec32ed4985a3e634c44d38c517564f5b │ 0x346c0b53faea3eaa013aebe3019b0bf09448aeb1 │ 0x00001a39 │ 5.7459274561e-08 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x995ff15d8712d4b727f388dd9814841b9063801bd53fe3baccea3bf0b5a1c9fa │ 40 │ 0xc7b7bccba82df66e9c219751fcf300d75fab5c05 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0xba677db7 │ 2418622215047 │ 2418622215047 │ call │ call │ 486358 │ 433983 │ 13 │ │ 0xc7b7bccba82df66e9c219751fcf300d75fab5c05 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0xba677db7 │ 2.418622215047e-06 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x995ff15d8712d4b727f388dd9814841b9063801bd53fe3baccea3bf0b5a1c9fa │ 40 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0x94a365ca808029af8db18257ecd296c16c61ac05 │ 0x │ 2418622215047 │ 2418622215047 │ call │ call │ 377551 │ 0 │ 0 │ 3 │ 0xc7b7bccba82df66e9c219751fcf300d75fab5c05 │ 0xa2a786ff9148f7c88ee93372db8cbe9e94585c74 │ 0xba677db7 │ 2.418622215047e-06 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xe53957d3c4e3e819bbeb522bd1cf097f3c30abfe65f2a51303534ca00e141757 │ 39 │ 0x5d7e4ca0cd3d9aa42724a61d139cd93cbbadf825 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0xfc180638 │ 1557961691647504 │ 1557961691647504 │ call │ call │ 38283 │ 37917 │ 1 │ │ 0x5d7e4ca0cd3d9aa42724a61d139cd93cbbadf825 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0xfc180638 │ 0.001557961691647504 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xe53957d3c4e3e819bbeb522bd1cf097f3c30abfe65f2a51303534ca00e141757 │ 39 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0x5e809a85aa182a9921edd10a4163745bb3e36284 │ 0x │ 1557961691647504 │ 1557961691647504 │ call │ call │ 4897 │ 0 │ 0 │ 0 │ 0x5d7e4ca0cd3d9aa42724a61d139cd93cbbadf825 │ 0x0e83ded9f80e1c92549615d96842f5cb64a08762 │ 0xfc180638 │ 0.001557961691647504 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x8adc1668c8b8a55c6866e791eb70c343e33d89043c79b3bc19e9f74746596dfb │ 22 │ 0xf32314be92973a6d7c7ec60cdf114fa331e8f8e5 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x29723511 │ 380008468139144441 │ 380008468139144441 │ call │ call │ 58738 │ 45132 │ 1 │ │ 0xf32314be92973a6d7c7ec60cdf114fa331e8f8e5 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x29723511 │ 0.38000846813914446 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0x8adc1668c8b8a55c6866e791eb70c343e33d89043c79b3bc19e9f74746596dfb │ 22 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x80c67432656d59144ceff962e8faf8926599bcf8 │ 0x │ 380008468139144441 │ 380008468139144441 │ call │ call │ 2300 │ 0 │ 0 │ 0 │ 0xf32314be92973a6d7c7ec60cdf114fa331e8f8e5 │ 0x3191f40de6991b1bb1f61b7cec43d62bb337786b │ 0x29723511 │ 0.38000846813914446 │\n", - "│ mainnet │ 10 │ op │ 2024-10-30 │ 1730250603 │ 127325913 │ 0x6a338444de812763401d4065f5754f30f397bca556bfbdfa9425eae3c5077721 │ 0xc1cddca85df209a6c9eaca445001a2592f69c8a53d92618832ab42b095765f78 │ 23 │ 0x0000000071727de22e5e9d8baf0edac6f37da032 │ 0x433702873e33d4846d399a75aaf96eacf181d0b2 │ 0x │ 1140653506944 │ 1140653506944 │ call │ call │ 992723 │ 0 │ 0 │ 3 │ 0x433702873e33d4846d399a75aaf96eacf181d0b2 │ 0x0000000071727de22e5e9d8baf0edac6f37da032 │ 0x765e827f │ 1.140653506944e-06 │\n", - "├─────────┴──────────┴─────────┴────────────┴─────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┴────────────────────────────────────────────────────────────────────┴───────────────────┴────────────────────────────────────────────┴────────────────────────────────────────────┴─────────────────┴────────────────────┴────────────────────┴────────────┴───────────┴─────────────────┴────────────────┴───────────┴───────────────┴────────────────────────────────────────────┴────────────────────────────────────────────┴──────────────┴──────────────────────┤\n", - "│ 10 rows 24 columns │\n", - "└────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "duckdb_client.sql(\"SELECT *, value_64/1e18 AS value_native FROM native_transfers_v1 LIMIT 10\")" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### You can also convert the results to dataframes to inspect them in more familiar ways" + "duckdb_client.sql(\"SELECT * FROM erc20_transfers_v1 LIMIT 10\")" ] }, { - "cell_type": "code", - "execution_count": 4, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "shape: (5, 23)
networkchain_idchaindtblock_timestampblock_numberblock_hashtransaction_hashtransaction_indextransfer_from_addresstransfer_to_addresstrace_method_idvalue_64value_losslesstrace_typecall_typetrace_gas_limittrace_gas_usedsubtracestrace_addresstx_from_addresstx_to_addresstx_method_id
stri32strdateu32i64strstri64strstrstri64strstrstri64i64i64strstrstrstr
"mainnet"10"op"2024-10-301730250601127325912"0x091e27e20a77c42779efb7c7af67…"0xaf039d9965ee1ed1a047d06fe80f…7"0x1632ed73c1a866fe2c21e6fd0f49…"0xe4edb277e41dc89ab076a1f049f4…"0x"269375236255299057"269375236255299057""call""call"21000210000"""0x1632ed73c1a866fe2c21e6fd0f49…"0xe4edb277e41dc89ab076a1f049f4…"0x"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0xf958157473bb2d066d8225ec07ec…34"0x4200000000000000000000000000…"0x1c6858664ccfacc2dcc8175f663c…"0x"57459274561"57459274561""call""call"2300750"26,6,3,0""0x3c9779d3ec32ed4985a3e634c44d…"0x346c0b53faea3eaa013aebe3019b…"0x00001a39"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0xf958157473bb2d066d8225ec07ec…34"0x1c6858664ccfacc2dcc8175f663c…"0xabe58b50367fb4cea7d19c41c228…"0x"57459274561"57459274561""call""call"230000"26,6,4""0x3c9779d3ec32ed4985a3e634c44d…"0x346c0b53faea3eaa013aebe3019b…"0x00001a39"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0x995ff15d8712d4b727f388dd9814…40"0xc7b7bccba82df66e9c219751fcf3…"0xa2a786ff9148f7c88ee93372db8c…"0xba677db7"2418622215047"2418622215047""call""call"48635843398313"""0xc7b7bccba82df66e9c219751fcf3…"0xa2a786ff9148f7c88ee93372db8c…"0xba677db7"
"mainnet"10"op"2024-10-301730250603127325913"0x6a338444de812763401d4065f575…"0x995ff15d8712d4b727f388dd9814…40"0xa2a786ff9148f7c88ee93372db8c…"0x94a365ca808029af8db18257ecd2…"0x"2418622215047"2418622215047""call""call"37755100"3""0xc7b7bccba82df66e9c219751fcf3…"0xa2a786ff9148f7c88ee93372db8c…"0xba677db7"
" - ], - "text/plain": [ - "shape: (5, 23)\n", - "┌─────────┬──────────┬───────┬────────────┬───┬─────────────┬────────────┬────────────┬────────────┐\n", - "│ network ┆ chain_id ┆ chain ┆ dt ┆ … ┆ trace_addre ┆ tx_from_ad ┆ tx_to_addr ┆ tx_method_ │\n", - "│ --- ┆ --- ┆ --- ┆ --- ┆ ┆ ss ┆ dress ┆ ess ┆ id │\n", - "│ str ┆ i32 ┆ str ┆ date ┆ ┆ --- ┆ --- ┆ --- ┆ --- │\n", - "│ ┆ ┆ ┆ ┆ ┆ str ┆ str ┆ str ┆ str │\n", - "╞═════════╪══════════╪═══════╪════════════╪═══╪═════════════╪════════════╪════════════╪════════════╡\n", - "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ ┆ 0x1632ed73 ┆ 0xe4edb277 ┆ 0x │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ c1a866fe2c ┆ e41dc89ab0 ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ 21e6fd0f49 ┆ 76a1f049f4 ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", - "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ 26,6,3,0 ┆ 0x3c9779d3 ┆ 0x346c0b53 ┆ 0x00001a39 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ec32ed4985 ┆ faea3eaa01 ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ a3e634c44d ┆ 3aebe3019b ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", - "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ 26,6,4 ┆ 0x3c9779d3 ┆ 0x346c0b53 ┆ 0x00001a39 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ ec32ed4985 ┆ faea3eaa01 ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ a3e634c44d ┆ 3aebe3019b ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", - "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ ┆ 0xc7b7bccb ┆ 0xa2a786ff ┆ 0xba677db7 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ a82df66e9c ┆ 9148f7c88e ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ 219751fcf3 ┆ e93372db8c ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", - "│ mainnet ┆ 10 ┆ op ┆ 2024-10-30 ┆ … ┆ 3 ┆ 0xc7b7bccb ┆ 0xa2a786ff ┆ 0xba677db7 │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ a82df66e9c ┆ 9148f7c88e ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ 219751fcf3 ┆ e93372db8c ┆ │\n", - "│ ┆ ┆ ┆ ┆ ┆ ┆ … ┆ … ┆ │\n", - "└─────────┴──────────┴───────┴────────────┴───┴─────────────┴────────────┴────────────┴────────────┘" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "duckdb_client.sql(\"SELECT * FROM native_transfers_v1 LIMIT 10\").pl().head()" + "### You can also convert the results to dataframes to inspect them in more familiar ways" ] }, { @@ -230,7 +108,9 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "duckdb_client.sql(\"SELECT * FROM native_transfers_v1 LIMIT 10\").pl().head()" + ] } ], "metadata": { diff --git a/src/op_analytics/datapipeline/models/code/token_transfers.py b/src/op_analytics/datapipeline/models/code/token_transfers.py index 5ede37295bd..6c39126f995 100644 --- a/src/op_analytics/datapipeline/models/code/token_transfers.py +++ b/src/op_analytics/datapipeline/models/code/token_transfers.py @@ -1,41 +1,56 @@ import duckdb +from duckdb.typing import VARCHAR, BIGINT # Import DuckDB's type annotations from op_analytics.datapipeline.models.compute.querybuilder import TemplatedSQLQuery from op_analytics.datapipeline.models.compute.registry import register_model from op_analytics.datapipeline.models.compute.types import NamedRelations +from eth_abi import decode @register_model( input_datasets=["ingestion/traces_v1", "ingestion/logs_v1", "ingestion/transactions_v1"], - expected_outputs=["native_transfers_v1"], + expected_outputs=["native_transfers_v1", "erc20_transfers_v1"], auxiliary_views=[ - TemplatedSQLQuery( - template_name="native_transfers", - context={}, - ), - # TemplatedSQLQuery( - # template_name="erc20_transfers", - # context={}, - # ), + TemplatedSQLQuery(template_name="native_transfers", context={}), + TemplatedSQLQuery(template_name="filtered_raw_erc20_transfer_logs", context={}), ], ) def token_transfers(duckdb_client: duckdb.DuckDBPyConnection) -> NamedRelations: - # Generic Function to decode events with the ABI - Will Prototype in dev notebook? - # def decode_logs(df): - # Get Unique Topic0s - # Grab ABIs for Topic0s - From some log we have? Maybe look up to a repo eventually? - # Decode the appropriate row based on Topic0 <> ABI Mapping - # Retrun Decoded Table - # return - # Get raw filtered logs for each token standard - # erc20_view = duckdb_client.view("erc20_transfer_raw_logs") - # erc721_view = duckdb_client.view("erc721_transfer_raw_logs") - # erc1155_view = duckdb_client.view("erc1155_transfer_raw_logs") - - # decoded_erc20_view = decode_events(erc20_view, abi) + def decode_erc20_transfer(topics: str, data: str): + """ + Decode ERC-20 transfer log data into structured fields. + :param topics: JSON-like string representing log topics. + :param data: Hexadecimal string of ABI-encoded data. + :return: Tuple of (from_address, to_address, value). + """ + import json + + topics = json.loads(topics) # Convert JSON string to list + if len(topics) < 3: + raise ValueError("Insufficient topics to decode transfer.") + + from_address = decode(["address"], bytes.fromhex(topics[1][2:]))[0] + to_address = decode(["address"], bytes.fromhex(topics[2][2:]))[0] + value = decode(["uint256"], bytes.fromhex(data[2:]))[0] + return from_address, to_address, value + + # Register the function in DuckDB + duckdb_client.create_function( + "decode_erc20_transfer", + decode_erc20_transfer, + parameters=[VARCHAR, VARCHAR], # Input types + return_type=[VARCHAR, VARCHAR, BIGINT], # Output types: 3 separate columns + ) + + erc20_logs_decoded = duckdb_client.view(""" + SELECT + decode_erc20_transfer(topics, data).from_address AS from_address, + decode_erc20_transfer(topics, data).to_address AS to_address, + decode_erc20_transfer(topics, data).value AS value + FROM filtered_raw_erc20_transfer_logs l + """) return { - # TBD if we materialize these individually, or roll them up in to a token transfers table "native_transfers_v1": duckdb_client.view("native_transfers"), - # "erc20_transfers_v1": decoded_erc20_view, + "erc20_transfers_v1": erc20_logs_decoded, } diff --git a/src/op_analytics/datapipeline/models/compute/udfs.py b/src/op_analytics/datapipeline/models/compute/udfs.py index 82172bde0e5..4bfb56ceb05 100644 --- a/src/op_analytics/datapipeline/models/compute/udfs.py +++ b/src/op_analytics/datapipeline/models/compute/udfs.py @@ -65,7 +65,7 @@ def create_duckdb_macros(duckdb_client: duckdb.DuckDBPyConnection): --Get the method id for input data. This is the first 4 bytes, or first 10 -- string characters for binary data that is encoded as a hex string. CREATE OR REPLACE MACRO hexstr_method_id(x) - AS substring(x,1,10) + AS substring(x,1,10); """) diff --git a/src/op_analytics/datapipeline/models/templates/filtered_raw_erc20_transfer_logs.sql.j2 b/src/op_analytics/datapipeline/models/templates/filtered_raw_erc20_transfer_logs.sql.j2 new file mode 100644 index 00000000000..3c0091e47b3 --- /dev/null +++ b/src/op_analytics/datapipeline/models/templates/filtered_raw_erc20_transfer_logs.sql.j2 @@ -0,0 +1,16 @@ +SELECT + l.* + , t.from_address AS tx_from_address + , t.to_address AS tx_to_address + , hexstr_method_id(t.input) AS tx_method_id + +FROM ingestion_logs_v1 AS l +INNER JOIN ingestion_transactions_v1 AS t + ON + l.block_number = t.block_number + AND l.chain_id = t.chain_id + AND l.transaction_hash = t.hash + +WHERE + l.topic0 = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + AND t.receipt_status = 1 -- only successful transactions emit event logs diff --git a/src/op_analytics/datapipeline/models/templates/filtered_raw_logs.sql.j2 b/src/op_analytics/datapipeline/models/templates/filtered_raw_logs.sql.j2 new file mode 100644 index 00000000000..72d1e2281b5 --- /dev/null +++ b/src/op_analytics/datapipeline/models/templates/filtered_raw_logs.sql.j2 @@ -0,0 +1,14 @@ +SELECT + l.* + , t.from_address AS tx_from_address + , t.to_address AS tx_to_address + , hexstr_method_id(t.input) AS tx_method_id + +FROM ingestion_logs_v1 AS l +INNER JOIN ingestion_transactions_v1 AS t + ON + l.block_number = t.block_number + AND l.chain_id = t.chain_id + AND l.transaction_hash = t.hash +WHERE + l.topic0 = '{{ topic0_filter }}' diff --git a/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 b/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 index 1da74eef51d..69d54c35c92 100644 --- a/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 +++ b/src/op_analytics/datapipeline/models/templates/native_transfers.sql.j2 @@ -5,7 +5,6 @@ SELECT , tr.dt , tr.block_timestamp , tr.block_number - , tr.block_hash , tr.transaction_hash , tr.transaction_index , tr.from_address AS transfer_from_address