peerdb-io / peerdb Goto Github PK

peerdb-stack-peerdb-1       | 2023-06-12T17:17:21.339614Z  INFO Migration Applied -  Name: peer_connections, Version: 2
peerdb-stack-peerdb-1       | 2023-06-12T17:17:21.339617Z  INFO Migration Applied -  Name: add_workflow_id_to_flows, Version: 3
peerdb-stack-peerdb-1       | 2023-06-12T17:17:21.340716Z  INFO Listening on 0.0.0.0:9900
peerdb-stack-peerdb-1       | Error: failed to send health check request
peerdb-stack-peerdb-1       |
peerdb-stack-peerdb-1       | Caused by:
peerdb-stack-peerdb-1       |     0: error sending request for url (http://flow_api:8112/health): error trying to connect: dns error: failed to lookup address information: Name or service not known
peerdb-stack-peerdb-1       |     1: error trying to connect: dns error: failed to lookup address information: Name or service not known
peerdb-stack-peerdb-1       |     2: dns error: failed to lookup address information: Name or service not known
peerdb-stack-peerdb-1       |     3: failed to lookup address information: Name or service not known
peerdb-stack-peerdb-1 exited with code 1```

CREATE TABLE test_types_issue_1 (
    c1 UUID NOT NULL PRIMARY KEY,
    c2 UUID,
    "from" TIMESTAMP NOT NULL,
    c4 NUMERIC,
    c5 TIMESTAMP NOT NULL,
    c6 TIMESTAMP NOT NULL,
    c7 BYTEA,
    c8 VARCHAR,
    c9 UUID,
    c10 INTEGER,
    c11 INTEGER DEFAULT 0 NOT NULL,
    c12 INTEGER NOT NULL,
    c13 VARCHAR,
    c14 UUID,
    c15 UUID,
    c16 BOOLEAN DEFAULT false,
    c17 DOUBLE PRECISION,
    c18 DOUBLE PRECISION,
    c19 BOOLEAN DEFAULT false NOT NULL,
    c20 NUMERIC,
    c21 UUID,
    c22 NUMERIC NOT NULL,
    c23 INTEGER,
    c24 UUID,
    c25 TIMESTAMP,
    c26 VARCHAR,
    c27 TIMESTAMP,
    c28 INTEGER
);

CREATE TABLE test_null(id int);
INSERT INTO test_null VALUES (NULL);

admin=> SELECT * FROM test_null;
server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
The connection to the server was lost. Attempting reset: Succeeded.
psql (15.2 (Debian 15.2-1.pgdg110+1), server 14)

-- CREATE POSTGRES PEER
CREATE PEER postgres_peer FROM POSTGRES WITH
(
    host = '34.162.117.149',
    port = '5432',
    user = 'postgres',
    password = 'peerdb123#!#',
    database = 'postgres'
);

1. SELECT array_agg(c1) FROM my_bq_b.test_types LIMIT 1;
2. SELECT array_agg(c2) FROM my_bq_b.test_types LIMIT 1;
3. SELECT array_agg(c3) FROM my_bq_b.test_types LIMIT 1;
4. SELECT array_agg(c13) FROM my_bq_b.test_types LIMIT 1;
5. SELECT array_agg(c20) FROM my_bq_b.test_types LIMIT 1;
6. SELECT count(*) FROM my_bq_b.transactions where tx_timestamp > '2022-09-07 14:42:49.105761-07'::timestamp - interval '3 months'; ➡️ SELECT count(*) FROM test_dataset.transactions where tx_timestamp > DATE_SUB(CAST('2022-09-07 14:42:49.105761-07' AS TIMESTAMP),interval 90 DAY); 
7. SELECT count(*) FROM my_bq_b.transactions where tx_timestamp > '2022-09-07 14:42:49.105761-07'::timestamp - interval '3 days';
8. SELECT * FROM my_bq_b.transactions where tx_timestamp >now() - interval '3 months';➡️SELECT * FROM test_dataset.transactions where tx_timestamp >DATE_SUB(CURRENT_TIMESTAMP,interval 90 DAY);
9. SELECT date_trunc('month',tx_timestamp),count(*) FROM my_bq_b.transactions GROUP BY 1;➡️SELECT date_trunc(tx_timestamp,MONTH),count(*) FROM test_dataset.transactions GROUP BY 1;
10. SELECT array_agg(chain) FROM my_bq_b.transactions; ➡️ Same as above - SELECT array_agg(chain) FROM test_dataset.transactions;
11. SELECT COALESCE(chain,'sai'::text) FROM my_bq_b.transactions;➡️SELECT COALESCE(chain,'sai') FROM test_dataset.transactions;
12. SELECT * FROM my_bq_b.transactions UNION SELECT * FROM my_bq_b.transactions;➡️ SELECT * FROM test_dataset.transactions UNION DISTINCT SELECT * FROM test_dataset.transactions;
13. SELECT NULL FROM my_bq_b.transactions
14. SELECT count(*) FROM (my_bq_b.users r1 INNER JOIN my_bq_b.events r2 ON (((r2.user_id = 1)) AND ((r1.id = 1))));➡️ SELECT count(*) FROM (test_dataset.users r1 INNER JOIN test_dataset.events r2 ON (((r2.user_id = 1)) AND ((r1.id = 1))));

curl --location 'http://localhost:8112/qrep/start' \
--header 'Content-Type: application/json' \
--data-raw '{
    "flow_job_name": "7_owner_flow_rat_fix",
    "source_peer": {
        "name": "test_postgres_peer",
        "type": 3,
        "postgres_config": {
            "host": "host.docker.internal",
            "port": 7132,
            "user": "postgres",
            "password": "<>",
            "database": "postgres"
        }
    },
    "destination_peer": {
        "name": "test_bq_peer",
        "bigquery_config": {
            "auth_type": "service_account",
            "project_id": "custom-program-353117",
            "private_key_id": "<>",
            "private_key": "<>",
            "client_email": "<>",
            "client_id": "<>,
            "auth_uri": "<>",
            "token_uri": "<>",
            "auth_provider_x509_cert_url": "<>",
            "client_x509_cert_url": "<>",
            "dataset_id": "<>"
        }
    },
    "destination_table_identifier": "<>", (destination_table_name)
    "watermark_table":"<>", (watermark_table_name)
    "query": "<>",
    "watermark_column": "<>",
    "inital_copy_only": false, (always false)
    "sync_mode": 1, (sync_format, 1 is avro and 0 is default)
    "wait_between_batches_seconds": <>, (get this from refresh_interval in minutes and send it in seconds)
    "max_parallel_workers":<>, (parallelism)
    "batch_size_int":<>, (batch size)
    "write_mode" : { "write_type": 0 (append) or 1 (upsert), "upsert_key_columns" : [ "k1", "k2" ] (only if write_type is 1) }
}'

-- connect to catalog
 CREATE DATABASE fdw_test;
 \c fdw_test
 CREATE EXTENSION postgres_fdw;
 
 CREATE SERVER bigquery FOREIGN DATA WRAPPER postgres_fdw OPTIONS(host 'host.docker.internal', port '9900');
 
 CREATE USER MAPPING FOR postgres SERVER bigquery OPTIONS(user 'peerdb',password 'peerdb');
 
 CREATE FOREIGN TABLE test_fdw (id int, t1 text, t2 text) SERVER bigquery 
 OPTIONS(schema_name '<your_peer_name>',table_name 'test'); -- assumes that the table test is present in your peer.
 
   SELECT * FROM test_fdw;
 ERROR:  could not obtain message string for remote error
CONTEXT:  remote SQL command: SELECT id, t1, t2 FROM sai_fresh_bq.test

---- extended_query_protocol_no_params_catalog stdout ----
Starting server...
peerdb-server Server started
thread 'extended_query_protocol_no_params_catalog' panicked at 'Failed to prepare query: Error { kind: UnexpectedMessage, cause: None }', server/tests/server_test.rs:192:10
stack backtrace:
   0: rust_begin_unwind
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
   1: core::panicking::panic_fmt
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
   2: core::result::unwrap_failed
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/result.rs:1750:5
   3: core::result::Result<T,E>::expect
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/result.rs:1047:23
   4: server_test::extended_query_protocol_no_params_catalog
             at ./tests/server_test.rs:190:16
   5: server_test::extended_query_protocol_no_params_catalog::{{closure}}
             at ./tests/server_test.rs:185:48
   6: core::ops::function::FnOnce::call_once
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/ops/function.rs:250:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
Stopping server...
Server stopped

{
  "message": "failed to cleanup source: error dropping publication: ERROR: publication \"peerflow_pub_test10\" does not exist (SQLSTATE 42704)",
  "source": "GoSDK",
  "stackTrace": "",
  "encodedAttributes": null,
  "cause": {
    "message": "error dropping publication: ERROR: publication \"peerflow_pub_test10\" does not exist (SQLSTATE 42704)",
    "source": "GoSDK",
    "stackTrace": "",
    "encodedAttributes": null,
    "cause": {
      "message": "ERROR: publication \"peerflow_pub_test10\" does not exist (SQLSTATE 42704)",
      "source": "GoSDK",
      "stackTrace": "",
      "encodedAttributes": null,
      "cause": null,
      "applicationFailureInfo": {
        "type": "PgError",
        "nonRetryable": false,
        "details": null
      }
    },
    "applicationFailureInfo": {
      "type": "wrapError",
      "nonRetryable": false,
      "details": null
    }
  },
  "applicationFailureInfo": {
    "type": "wrapError",
    "nonRetryable": false,
    "details": null
  }
}

  "message": "failed to get partitions from source: unsupported type: <nil>",
  "source": "GoSDK",
  "stackTrace": "",
  "encodedAttributes": null,
  "cause": {
    "message": "unsupported type: <nil>",
    "source": "GoSDK",
    "stackTrace": "",
    "encodedAttributes": null,
    "cause": null,
    "applicationFailureInfo": {
      "type": "",
      "nonRetryable": false,
      "details": null
    }
  },
  "applicationFailureInfo": {
    "type": "wrapError",
    "nonRetryable": false,
    "details": null
  }
}

2023-05-18 20:44:58.593 PDT [11501] FATAL:  sorry, too many clients already
2023-05-18 20:44:58.597 PDT [11407] LOG:  could not receive data from client: Connection reset by peer
2023-05-18 20:44:58.603 PDT [11441] LOG:  could not receive data from client: Connection reset by peer
2023-05-18 20:44:58.603 PDT [11447] LOG:  could not receive data from client: Connection reset by peer
2023-05-18 20:44:58.605 PDT [11471] LOG:  could not receive data from client: Connection reset by peer
pgbench: error: client 22 script 0 aborted in command 2 query 0: ERROR:  connection closed
pgbench: pgbench: error: client 12 script 0 aborted in command 2 query 0: ERROR:  connection closed
error: client 45 script 0 aborted in command 2 query 0: ERROR:  connection closed
pgbench: error: client 41 script 0 aborted in command 2 query 0: ERROR:  connection closed
pgbench: error: client 27 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 47 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 44 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 25 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 1 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 36 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 40 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 19 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 0 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 11 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 35 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 4 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 37 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 49 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 9 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 23 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 6 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 2 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 32 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 16 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 43 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 18 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 15 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 5 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 33 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 42 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 39 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 7 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 14 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 20 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 30 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 21 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 24 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 26 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 10 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 34 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 13 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 48 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 46 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 38 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 31 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 8 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 29 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 3 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 28 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
pgbench: error: client 17 aborted in command 2 (SQL) of script 0; perhaps the backend died while processing
transaction type: 1.sql
scaling factor: 1
query mode: simple
number of clients: 50
number of threads: 50
number of transactions per client: 10
number of transactions actually processed: 0/500
pgbench: fatal: Run was aborted; the above results are incomplete.

Error: Failed to connect to catalog database

Caused by:
    0: db error: FATAL: sorry, too many clients already
    1: FATAL: sorry, too many clients already