GithubHelp home page GithubHelp logo

konnexionsgmbh / oranif Goto Github PK

View Code? Open in Web Editor NEW
9.0 9.0 2.0 742 KB

High performance Oracle DB driver for erlang

License: Other

Makefile 0.57% C 48.42% Erlang 50.86% Shell 0.15%
database driver erlang nif oracle

oranif's People

Contributors

acautin avatar c-bik avatar karlkeiser avatar nkezhaya avatar shamis avatar walter-weinmann avatar

Stargazers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

Watchers

 avatar  avatar  avatar  avatar  avatar  avatar

Forkers

sparkworx

oranif's Issues

Git Action to run coverage

TBD:

  1. apt-get install lcov in dderl_dev container (FYI @walter-weinmann)
  2. @c-bik to provide a script to error out when overall coverage below parameterized coverage
  3. @shamis add konnexionsw/ora_bench_db from docker hub into action (or another way to start a oracle DB)
  4. @c-bik / @walter-weinmann to find a to dynamically define Oracle DB IP into rebar3 eunit config

I wanted to add all of us assigned to it but current git subscription doesn't allow that

`rebar3 eunit` in docker fails

cover_tests: load_test...[0.120 s] ok
cover_tests: slave_reuse_test...[ERROR] REVISIT dpi:135 boot file not found!!
=ERROR REPORT==== 28-Apr-2020::14:11:19.564411 ===
global: <0.154.0> registered under several names: [{dpi,'[email protected]','[email protected]',#Ref<0.1343750239.56360961.202328>},{dpi,'[email protected]','[email protected]',#Ref<0.1343750239.56360961.201929>}]

*failed*
in function cover_tests:'-slave_reuse_test/0-fun-0-'/1 (/oranif/test/cover_tests.erl, line 1848)
in call from cover_tests:slave_reuse_test/0 (/oranif/test/cover_tests.erl, line 1848)
in call from eunit_test:'-mf_wrapper/2-fun-0-'/2 (eunit_test.erl, line 273)
in call from eunit_test:run_testfun/1 (eunit_test.erl, line 71)
in call from eunit_proc:run_test/1 (eunit_proc.erl, line 510)
in call from eunit_proc:with_timeout/3 (eunit_proc.erl, line 335)
in call from eunit_proc:handle_test/2 (eunit_proc.erl, line 493)
in call from eunit_proc:tests_inorder/3 (eunit_proc.erl, line 435)
**error:{assertEqual,[{module,cover_tests},
              {line,1848},
              {expression,"nodes ( hidden )"},
              {expected,[{error,"failed to register process globally"}]},
              {value,['[email protected]']}]}
  output:<<"">>

[done in 31.910 s]
module 'odpi_ct'
=======================================================
  Failed: 1.  Skipped: 0.  Passed: 7.
One or more tests were cancelled.

Connection Handling - Best Practice

Facing growing number of connections to the database.

My approach:

% connect:
%..
Context = dpi:context_create(?DPI_MAJOR_VERSION, ?DPI_MINOR_VERSION),
Connection = dpi:conn_create(
Context, User, Password, Tns,
#{encoding => Encoding, nencoding => Nencoding}, #{}
),

%..

%disconnect
ok = dpi:conn_close(Connection, [], <<>>),
ok = dpi:context_destroy(Context),

%

repeated execution of the above code leads to to a strictly monotonic growing number of database connections.

is there anything i missed?

thanks in advance for your asnwer.

my environmend:

os = debian 11
db: Oracle Database 18c Express Edition Release 18.0.0.0.0
erlang: otp24

CLOB Crashing

There's a crash when fetching a CLOB in some cases, as demonstrated by following escript:

#!/user/bin/escript
%% -*- erlang -*-
%%! -pa _build/default/lib/oranif/ebin

-include_lib("eunit/include/eunit.hrl").
-define(DPI_MAJOR_VERSION, 3).
-define(DPI_MINOR_VERSION, 0).
-define(TNS, <<>>).

fetch_all(Stmt, Acc) ->
     io:format("Oassl.~n"),
    #{found := Found} = dpi:stmt_fetch(Stmt),
    io:format("Found ~p.~n", [Found]),
    case Found of true ->
        io:format("Oassl 1.~n"),
        #{data := Result, nativeTypeNum := Type} = dpi:stmt_getQueryValue(Stmt, 1),
        D1 = dpi:data_get(Result),
        io:format("Oassl 3. ~p~n", [D1]),
        fetch_all(Stmt, Acc ++ [D1]);
    %fetch_all(Stmt, Acc ++ [dpi:data_get(Result)] );
    _ -> Acc end.


main([]) ->
	dpi:load_unsafe(),
	Context = dpi:context_create(?DPI_MAJOR_VERSION, ?DPI_MINOR_VERSION),
    Conn = dpi:conn_create(
        Context, <<"foo">>, <<"bar">>, ?TNS,
        #{encoding => "AL32UTF8", nencoding => "AL32UTF8"}, #{}
    ),

    StmtD2 = dpi:conn_prepareStmt(Conn, false, <<"drop table lob_test">>, <<>>),
    catch dpi:stmt_execute(StmtD2, ['DPI_MODE_EXEC_COMMIT_ON_SUCCESS']),
    SqlCreation = <<"CREATE TABLE
                    lob_test (a clob)">>,
    StmtC = dpi:conn_prepareStmt(Conn, false, SqlCreation, <<>>),
    0 = dpi:stmt_execute(StmtC, ['DPI_MODE_EXEC_COMMIT_ON_SUCCESS']),
    SQLInsert = <<"INSERT INTO
                    lob_test
                    values (to_clob(dbms_random.string('x', 1000000)))">>,
    StmtI = dpi:conn_prepareStmt(Conn, false, SQLInsert, <<>>),
    0 = dpi:stmt_execute(StmtI, ['DPI_MODE_EXEC_COMMIT_ON_SUCCESS']),
    Sql = <<"select * from lob_test">>,

    Stmt = dpi:conn_prepareStmt(Conn, true, Sql, <<>>),

    1 = dpi:stmt_execute(Stmt, ['DPI_MODE_EXEC_COMMIT_ON_SUCCESS']),
    LobSize = 3,

    #{var := Var, data := Datas} = dpi:conn_newVar(Conn, 'DPI_ORACLE_TYPE_CLOB', 'DPI_NATIVE_TYPE_LOB', 100, LobSize, false, false, null),
    ok = dpi:stmt_define(Stmt, 1, Var), 
    #{found := Found, bufferRowIndex := BRI} = dpi:stmt_fetch(Stmt),

    D1 = dpi:data_get(lists:nth(1, Datas)),
    Bytes = dpi:lob_readBytes(D1, 1, LobSize),
    io:format("Got data: ~p~n", [binary_to_list(Bytes)]),
    %[io:format("got em ~p~n", [X]) || X <- fetch_all(Stmt, [])],
    StmtD = dpi:conn_prepareStmt(Conn, false, <<"drop table lob_test">>, <<>>),
    0 = dpi:stmt_execute(StmtD, ['DPI_MODE_EXEC_COMMIT_ON_SUCCESS']),
    io:format("Test successful.~n"),

    % cleanup
    ok = dpi:stmt_close(StmtC, <<>>),
    ok = dpi:stmt_close(Stmt, <<>>),
    ok = dpi:stmt_close(StmtI, <<>>),
    ok = dpi:stmt_close(StmtD, <<>>),
    ok = dpi:conn_close(Conn, [], <<>>),
    ok = dpi:context_destroy(Context),
    halt(1).

Usually there's a segmentation fault, but sometimes I get this error:

image

I have no idea why the context would sometimes be wrong, this is likely just memory corruption.

The issue is with the LobSize (which would be set in ddconfig). There seems to be a huge problem with mismatching encodings that can cause the size to be wrong. Warning such as this one are in multiple place in the ODPI documentation:

image

After some testing I've noticed that the actual amount of characters read is equal to the LobSize given divided by 4. So it would seem that a character is stored as four bytes, at least on my machine. The crash is caused by setting the LobSize to less than 4.

I don't know how we can portably handle this problem. We could just multiply the LobSize by 4, but that would just work on my machine, and I don't know what else my influence it. The ODPI documentation implies that the encoding in the database has to do it, so things might be different in different versions of oracle.

Perhaps for now the best solution would be to just ensure that the LobSize isn't below 4 in dderl.

oranif master compiler warnings

@c-bik : Please comment on current oranif compiler warnings. Are they OK or addressed in unmerged branches?

===> Compiling oranif
mkdir -p priv
Cloning into 'odpi'...
Note: switching to '9902a60068042e7409b7806b5e73fe7e66200cae'.

You are in 'detached HEAD' state. You can look around, make experimental
...
Turn off this advice by setting config variable advice.detachedHead to false

Updating files: 100% (214/214), done.
gcc -o priv/dpi_nif.so -Ic_src -I"/usr/lib/erlang/erts-10.7.1/include" -I"c_src/odpi/include" -O2 -ggdb -Wall -fPIC -std=c11 -DEMBED c_src/dpiStmt_nif.c c_src/dpi_nif.c c_src/dpiVar_nif.c c_src/dpiContext_nif.c c_src/dpiConn_nif.c c_src/dpiData_nif.c -shared
In file included from c_src/dpiVar_nif.h:4,
from c_src/dpiStmt_nif.c:1:
c_src/dpiStmt_nif.c: In function â:
c_src/dpiStmt_nif.c:139:71: warning: pointer targets in passing argument 5 of â differ in signedness [-Wpointer-sign]
139 | stmtRes->stmt, maxRows, &bufferRowIndex, &numRowsFetched, &moreRows
| ^~~~~~~~~
| |
| uint32_t * {aka unsigned int *}
c_src/dpi_nif.h:157:25: note: in definition of macro â
157 | if (DPI_FAILURE == (_exprn))
| ^~~~~~
In file included from c_src/dpi_nif.h:6,
from c_src/dpiVar_nif.h:4,
from c_src/dpiStmt_nif.c:1:
c_src/odpi/include/dpi.h:1613:66: note: expected â but argument is of type â {aka â}
1613 | uint32_t *bufferRowIndex, uint32_t *numRowsFetched, int *moreRows);
| ~~~~~^~~~~~~~
c_src/dpiStmt_nif.c: In function â:
c_src/dpiStmt_nif.c:514:5: warning: â may be used uninitialized in this function [-Wmaybe-uninitialized]
514 | enif_make_map_put(
| ^~~~~~~~~~~~~~~~~~
515 | env, map, enif_make_atom(env, "statementType"), type, &map);
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from c_src/odpi/embed/dpi.c:23,
from c_src/dpi_nif.c:2:
c_src/odpi/embed/../src/dpiDebug.c: In function â:
c_src/odpi/embed/../src/dpiDebug.c:73:39: warning: implicit declaration of function â [-Wimplicit-function-declaration]
73 | threadId = (uint64_t) syscall(SYS_gettid);
| ^~~~~~~
c_src/odpi/embed/../src/dpiDebug.c:93:21: warning: implicit declaration of function â; did you mean â? [-Wimplic
it-function-declaration]
93 | localtime_r(&timeOfDay.tv_sec, &time);
| ^~~~~~~~~~~
| localtime

Compilation fails with OTP 24

Currently on macOS 11.1, installed with asdf:

src/dpi.erl:none: error in parse transform 'dpi_transform': {badarith,
                                           [{dpi_transform,nif_stubs,2,
                                             [{file,
                                               "deps/oranif/src/dpi_transform.erl"},
                                              {line,104}]},
                                            {dpi_transform,parse_transform,2,
                                             [{file,
                                               "deps/oranif/src/dpi_transform.erl"},
                                              {line,70}]},
                                            {compile,foldl_transform,3,
                                             [{file,"compile.erl"},
                                              {line,1145}]},
                                            {compile,fold_comp,4,
                                             [{file,"compile.erl"},
                                              {line,416}]},
                                            {compile,internal_comp,5,
                                             [{file,"compile.erl"},
                                              {line,400}]},
                                            {compile,
                                             '-internal_fun/2-anonymous-0-',2,
                                             [{file,"compile.erl"},
                                              {line,229}]},
                                            {compile,
                                             '-do_compile/2-anonymous-0-',1,
                                             [{file,"compile.erl"},
                                              {line,219}]}]}

SLAVE-DOWN issue on DBA_JOBS

For odpi on sbs1.int

SELECT
    WHAT, JOB, SCHEMA_USER, LAST_DATE, THIS_DATE, NEXT_DATE, TOTAL_TIME, BROKEN,
    FAILURES
FROM 
    DBA_JOBS
ORDER BY
    WHAT

grafik

{slave_down,[
{dderlodpi,run_query,6,[{file,"dderl/src/dderlodpi.erl"},{line,402}]},
{dderlodpi,exec,4,[{file,"dderl/src/dderlodpi.erl"},{line,79}]},
{odpi_adapter,process_query,3,[{file,"dderl/src/odpi_adapter.erl"},{line,742}]},
{odpi_adapter,process_cmd,6,[{file,"dderl/src/odpi_adapter.erl"},{line,307}]},
{dderl_session,spawn_process_call,8,[{file,"dderl/src/dderl_session.erl"},{line,659}]}]}

2020-04-29 18:27:37.932 [error] [dderl<0.8637.6>@odpi_adapter:process_query:777]
 query error {error,slave_down} for
 <<"SELECT\r\n WHAT, JOB, SCHEMA_USER, LAST_DATE, THIS_DATE, NEXT_DATE, TOTAL_TIME, BROKEN,\r\n FAILURES\r\nFROM\r\n DBA_JOBS\r\nORDER BY\r\n WHAT">>
 whith bind values undefined

2020-04-29 18:28:33.417 [error] [dderl<0.9902.6>@dderlodpi:exec:81]
 run_query(select CON_ID, CON_ACID, CON_ESTID, CON_NAME, CON_ESID, CON_DATESTART, CON_DATEEND, CON_SHORTID, CON_PMID from contract where CON_PMID in ('0259630091', '0559477345') and CON_SHORTID like '6%' and CON_ESID = 'A' and CON_ETID = 'IPC',undefined,select CON_ID, CON_ACID, CON_ESTID, CON_NAME, CON_ESID, CON_DATESTART, CON_DATEEND, CON_SHORTID, CON_PMID, contract.ROWID from contract where CON_PMID in ('0259630091', '0559477345') and CON_SHORTID like '6%' and CON_ESID = 'A' and CON_ETID = 'IPC')
{slave_down,[
{dderlodpi,run_query,6,[{file,"dderl/src/dderlodpi.erl"},{line,402}]},
{dderlodpi,exec,4,[{file,"dderl/src/dderlodpi.erl"},{line,79}]},
{odpi_adapter,process_query,3,[{file,"dderl/src/odpi_adapter.erl"},{line,746}]},
{odpi_adapter,process_cmd,6,[{file,"dderl/src/odpi_adapter.erl"},{line,357}]},
{dderl_session,spawn_process_call,8,[{file,"dderl/src/dderl_session.erl"},{line,659}]}]}

2020-04-29 18:28:33.417 [error] [dderl<0.9902.6>@odpi_adapter:process_query:777]
 query error {error,slave_down} for <<"select CON_ID, CON_ACID, CON_ESTID, CON_NAME, CON_ESID, CON_DATESTART, CON_DATEEND, CON_SHORTID, CON_PMID from contract where CON_PMID in ('0259630091', '0559477345') and CON_SHORTID like '6%' and CON_ESID = 'A' and CON_ETID = 'IPC'">> whith bind values []

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.