Changeset 37c5882 in tol

Timestamp:

Feb 18, 2016, 12:17:42 PM (11 years ago)

Author:

Jorge Pérez <josp.jorge@…>

Branches:

1941-docker-image-to-test, devel, master

Children:

a1c9e632

Parents:

9c4daac

Message:

fixes #1913, se reimplementa la lógica para extraer las cadenas de caracteres muy grandes. SQlGetData retorna SQL_NO_DATA cuando ya no hay más datos por retornar. Correciones a los test.

git-svn-id: ​https://www.tol-project.org/svn/tolp/trunk@7235 be4846ae-f051-483a-831f-b64e7d08527f

Location:

tol/dbdrivers

Files:

• CMakeLists.txt (modified) (1 diff)
• libtolodbc.c (modified) (2 diffs)
• libtolsqlite.c (modified) (1 diff)
• tests/DBTestBigTextODBC.tol (modified) (3 diffs)
• tests/DBTestIni.tol (modified) (1 diff)

tol/dbdrivers/CMakeLists.txt

@@ +1 @@
+if( CMAKE_COMPILER_IS_GNUCC )
+  add_definitions( -Wimplicit-function-declaration )
+endif( )
+
 if( ODBC_FOUND )
   add_library( tolodbc0 SHARED libtolodbc.c )

tol/dbdrivers/libtolodbc.c

@@ -22 +22 @@
 
 #include <stdlib.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
@@ -686 +687 @@
 }
 
+//#define CHUNK_SIZE 16*1024
+#define CHUNK_SIZE 4*1024
+#define INITIAL_ARRAY_SIZE 10
+#define GROW_ARRAY_INCR 10
+
+typedef struct 
+{
+  uint32_t NumberOfChunks;
+  int32_t LastChunk;
+  SQLLEN SizeOfLastChunk;
+  char **ArrayOfChunks;
+  SQLLEN SizeOfChunk;
+} ChunkOfTextType;
+
+void ChunkOfTextType_Init( ChunkOfTextType *chunkData )
+{
+  uint32_t i;
+  chunkData->NumberOfChunks = INITIAL_ARRAY_SIZE;
+  chunkData->ArrayOfChunks = calloc( INITIAL_ARRAY_SIZE, sizeof( char* ) );
+  for( i = 0; i < INITIAL_ARRAY_SIZE; ++i )
+    {
+      chunkData->ArrayOfChunks[i] = NULL;
+    }
+  chunkData->LastChunk = -1;
+  chunkData->SizeOfLastChunk = 0;
+  chunkData->SizeOfChunk = CHUNK_SIZE;
+}
+
+void ChunkOfTextType_Release( ChunkOfTextType *chunkData )
+{
+  uint32_t i;
+  for( i = 0; i < chunkData->NumberOfChunks; ++i )
+    {
+      if( chunkData->ArrayOfChunks[i] )
+        {
+          free( chunkData->ArrayOfChunks[i] );
+          chunkData->ArrayOfChunks[i] = NULL;
+        }
+    }
+  free( chunkData->ArrayOfChunks );
+  chunkData->ArrayOfChunks = NULL;
+  chunkData->NumberOfChunks = 0;
+  chunkData->LastChunk = -1;
+  chunkData->SizeOfLastChunk = 0;
+}
+
+char * ChunkOfTextType_GetChunk( ChunkOfTextType *chunkData )
+{
+  chunkData->LastChunk++;
+  if ( chunkData->LastChunk == chunkData->NumberOfChunks )
+    {
+      /*
+        printf( "reallocating array of chunks: LastChunk = %ld\n", chunkData->LastChunk );*/
+      uint32_t sizeNew = chunkData->NumberOfChunks + GROW_ARRAY_INCR;
+      int32_t i;
+      chunkData->ArrayOfChunks = realloc( chunkData->ArrayOfChunks, 
+                                          sizeNew * sizeof( char* ) );
+      for( i = chunkData->LastChunk; i < sizeNew; ++i )
+        {
+          chunkData->ArrayOfChunks[i] = NULL;
+        }
+      chunkData->NumberOfChunks = sizeNew;
+    }
+  if ( !chunkData->ArrayOfChunks[ chunkData->LastChunk ] )
+    {
+      chunkData->ArrayOfChunks[ chunkData->LastChunk ] = calloc( CHUNK_SIZE, sizeof(char) );
+    }
+  return chunkData->ArrayOfChunks[ chunkData->LastChunk ];
+}
+
+void ChunkOfTextType_DiscardLast( ChunkOfTextType *chunkData )
+{
+  if ( chunkData->LastChunk >= 0 )
+    {
+      --chunkData->LastChunk;
+    }
+}
+
+void ChunkOfTextType_SetLastSize( ChunkOfTextType *chunkData )
+{
+  chunkData->SizeOfLastChunk = strlen( chunkData->ArrayOfChunks[ chunkData->LastChunk ] );
+}
+
+char *ChunkOfTextType_GetCopyFullBuffer( ChunkOfTextType *chunkData )
+{
+  uint32_t size = chunkData->LastChunk * (chunkData->SizeOfChunk - 1) + chunkData->SizeOfLastChunk;
+  char *buffer = malloc( size + 1), *ptr;
+  uint32_t i;
+
+  //printf( "GetCopyFullBuffer: size = %ld, LastChunk = %ld, SizeOfLastChunk = %ld\n", size, chunkData->LastChunk, chunkData->SizeOfLastChunk );
+  ptr = buffer;
+  for( i = 0; i < chunkData->LastChunk; ++i )
+    {
+      memcpy( ptr, chunkData->ArrayOfChunks[i], chunkData->SizeOfChunk - 1);
+      ptr += chunkData->SizeOfChunk - 1;
+    }
+  // copy last chunk
+  memcpy( ptr, chunkData->ArrayOfChunks[i], chunkData->SizeOfLastChunk );
+  buffer[ size ] = '\0';
+  return buffer;
+}
+
 //-------------------------------------------------------------------
 DLLEXPORT(int) odbc_GetAsText(odbcd *dbd, int n_field, UCHAR **txt_val)
 {
-  SQLRETURN status;
-  SQLINTEGER num_required=MAX_TEXT_FIELD_LENGTH;
-  SQLLEN num_available=0;
-  UCHAR *str, *long_str;
-
-  str = (UCHAR*) calloc(num_required, sizeof(UCHAR));
-  if(str==NULL) {
-    stdOutWriter("Out of memory retrieving text ODBC.\n");
-    return 0;
-  }
-  
-  status = SQLGetData(dbd->hstmt,n_field+1,SQL_CHAR,str,num_required,&num_available);
-  if((status!=SQL_SUCCESS) && (status!=SQL_SUCCESS_WITH_INFO)) { return 0; }
-  if(num_available==SQL_NULL_DATA) { return 1; }
-
-  if(status==SQL_SUCCESS_WITH_INFO)
-  {
-    long_str = (UCHAR*) calloc(num_available+1, sizeof(UCHAR));
-    if(long_str==NULL) {
-      stdOutWriter("Out of memory retrieving text from ODBC.\n");
-      return 0;
-    }
-    long_str = (unsigned char*) strncpy((char*)long_str,(char*)str,MAX_TEXT_FIELD_LENGTH-1);
-    free(str);
-    str = (UCHAR*) calloc((num_available+2)-num_required, sizeof(UCHAR));
-    if(str==NULL) {
-      stdOutWriter("Out of memory retrieving text from ODBC.\n");
-      return 0;
-    }
-    status = SQLGetData(dbd->hstmt,n_field+1,SQL_CHAR,str,num_available,&num_available);
-    long_str = (unsigned char*) strncat((char*)long_str, (char*)str, num_available);
-    free(str);
-    str = long_str;
-  }
-    
-  *txt_val = str;
+  ChunkOfTextType chunkData;
+  char *buffer = NULL;
+  SQLLEN LenOrInd;
+  SQLRETURN     rc; 
+
+  ChunkOfTextType_Init( &chunkData );
+
+  do
+    {
+      buffer = ChunkOfTextType_GetChunk( &chunkData );
+      rc = SQLGetData( dbd->hstmt, n_field+1, 
+                       SQL_C_CHAR, buffer, chunkData.SizeOfChunk,
+                       &LenOrInd );
+      //printf( "rc = %ld LenOrInd = %ld\n", rc, LenOrInd ); 
+      if ( rc == SQL_NO_DATA )
+        {
+          ChunkOfTextType_DiscardLast( &chunkData );
+          break;
+        }
+      if ( rc == SQL_ERROR )
+        {
+          ChunkOfTextType_Release( &chunkData );
+          return 0;
+        }
+      if ( LenOrInd == SQL_NO_TOTAL )
+        {
+          continue;
+        }
+      if ( LenOrInd ==  SQL_NULL_DATA )
+        {
+          // Esto solo puede pasar en el primer chunk
+          ChunkOfTextType_Release( &chunkData );
+          *txt_val = NULL;
+          return 1;
+        }
+      ChunkOfTextType_SetLastSize( &chunkData );
+    } while( 1 );
+  *txt_val = ChunkOfTextType_GetCopyFullBuffer( &chunkData );
+  ChunkOfTextType_Release( &chunkData );
   return 1;
 }

tol/dbdrivers/libtolsqlite.c

@@ -24 +24 @@
 #include <stdio.h>
 #include <string.h>
+#include <ctype.h>
 
 #define SQLFLOAT double

tol/dbdrivers/tests/DBTestBigTextODBC.tol

@@ -19 +19 @@
     Real res01 = If (DBExecQuery("drop table pruebas1") == 0, 1, 0);
     Real res02 = If (DBExecQuery("CREATE TABLE pruebas1 (
-                      `campo1` VARCHAR(1),
-                      `campo2` VARCHAR(255),
-                      `campo3` TEXT,
-                      `campo4` TEXT,
-                      `campo5` TEXT)
+                      campo1 VARCHAR(1),
+                      campo2 VARCHAR(255),
+                      campo3 TEXT,
+                      campo4 TEXT,
+                      campo5 TEXT)
                      ") == 0,1,0);
 
@@ -44 +44 @@
   Real TestRow(Set row)
   {
+    Set lengths = EvalSet( row, Real(Text field) {TextLength(field)} );
+    Set View( lengths, "" ); WriteLn( "" );
     Real (TextMatch(row[1],"A")&
           TextMatch(row[2],Repeat("ABCDE",25))&
@@ -67 +69 @@
   {
     Set ODBC   = DBStructODBC("odbc");
-    Real res01 = DBOpen(aliasNames[1],user,passwd, ODBC);
+    Real res01 = DBOpen(aliasNames[1],user,passwd);
 
     Real res02 = InitDB(0);
-    Real res03 = FillDB(100);
-    Real res04 = TestDB(100);
+    Real res03 = FillDB(1);
+    Real res04 = TestDB(1);
 
     Real res20 = DBClose(aliasNames[1]);

tol/dbdrivers/tests/DBTestIni.tol

@@ -6 +6 @@
 
 //If FALSE, no DB check will be done:
-Real testDB    = False;
+Real testDB    = True;
 
 // Same user/password for all test databases:
-Text user      = "user";
-Text passwd    = "password";
+Text user      = "toldevel";
+Text passwd    = "tol!116.";
 
 // Database name for native access:
-Text dbName    = "test";
+Text dbName    = "bysfortoldevel";
 
 //Host name for mysql access:
-Text host      = "localhost";
+Text host      = "t-mysql01.localbayes.es";
 
 // Aliasnames for two ODBC handlers:
-Set aliasNames = [["test2","test3"]];
+Set aliasNames = [["MySQLTolDevel", "OTAN","MSSQLTolDevel"]];