toaco / carry Goto Github PK

from sqlalchemy import Table, Column, MetaData, ForeignKey, Integer, String, DateTime, create_engine
from carry import exc
from datetime import datetime

metadata = MetaData()
source_table = Table(
'hh_sf_charge_code', metadata,
Column('charge_code', String),
Column('charge_name', String),
Column('pycode', String),
Column('unit_price', String),
Column('charge_unit', String),
Column('bill_code', String),
Column('Stop_flag', String),
)

target_table = Table(
'etl_feebas_tmp', metadata,
Column('id', Integer, primary_key=True),
Column('create_date', DateTime, nullable=False),
Column('fee_code', String),
Column('fee_desc', String),
Column('exe_pacode', String),
Column('fee_desc', String),
Column('unit_amt', String),
Column('fee_kind', String),
Column('disabled', String),
)

addresses = Table(
'addresses', metadata,
Column('id', Integer, primary_key=True),
Column('user_id', None, ForeignKey('users.id')),
Column('email_address', String, nullable=False, unique=True),
)

class DotDict(dict):
def getattr(self, item):
return self[item]

class Databases(DotDict):
def init(self, config, **kwargs):
super(Databases, self).init(**kwargs)
for name in config:
self[name] = DotDict({
'name': name,
'url': config[name],
'engine': create_engine(config[name])
})

import urllib
params = urllib.parse.quote_plus("DRIVER={SQL Server Native Client 11.0};SERVER=DESKTOP-CCEKAVK;DATABASE=pfzip;UID=sa;PWD=1234")

databases = Databases({
'db1': "mssql+pyodbc:///?odbc_connect=%s" % params, #'mssql+pyodbc://DESKTOP-CCEKAVK@localhost/pfzip', # + join_path(databases_dir, 'db2'),
'db2': "postgresql+psycopg2://openpg:openpgpwd@localhost/PFZIP", #'postgresql://openpg:openpgpwd@localhost/PFZIP', # + join_path(databases_dir, 'db1'),
})

db1 = databases.db1
db2 = databases.db2

from sqlalchemy import func, select

import carry

def test():
# Arrange
# source_table.create(db1.engine)
# target_table.create(db2.engine)
# db1.engine.execute(
# target_table.insert(),
# [
# {'name': 'wendy', 'fullname': 'Wendy Williams', 'reg_time': datetime.datetime.now()},
# {'name': 'wendy', 'fullname': 'Wendy Williams', 'reg_time': datetime.datetime.now()},
# {'name': 'wendy', 'fullname': 'Wendy Williams', 'reg_time': datetime.datetime.now()}
# ]
# )

# Act
config = {
    'STORES': [
        {
            'name': 'pfzip',
            'url': db1.url,
        },
        {
            'name': 'PFZIP',
            'url': db2.url,
        }
    ],
    'TASKS': [
        {
            'from': [{
                'name': 'pfzip'
            }],
            'to': {
                'name': 'PFZIP',
            },
            'orders': [
                'hh_sf_charge_code'
            ],
        }
    ]
}
carry.run(config)

carry.table(name='etl_feebas_tmp',
            header={
                'charge_code': 'fee_code',
                'charge_name': 'fee_desc',
                'py_code': 'exe_pacode',
                'unit_price': 'fee_desc',
                'charge_unit': 'unit_amt',
                'bill_code': 'fee_kind',
                'Stop_flag': 'disabled',
                'unknown': 'unknown',
            }, source_name='hh_sf_charge_code',
            )

# Assert
carry.run(config)
count = db2.engine.execute(select([func.count()]).select_from(target_table)).scalar()

test()

can this tool support streaming or extraction/loading by chunks?