/* Making a GAUSS dataset from ascii or excel data file. "; */

g_atog: ;
fhh=0;

"-----------------------------------------";
" Making a GAUSS dataset from ascii data file. ";


{fnames,finfo}=fileinfo(make_in);
if finfo==0 ;
  call grbl2_error( "Error opening ascii data file: " $+ make_in,1) ;
  stop;
endif;



" Ascii (text) input file= " ;; $make_in ;
format /ld 10,0 ; " Size= ";;g_addcomma(finfo[1,8],0)    ;;
bb=utctodt(finfo[1,10]);
bb=dttostr(bb,"YYYY-MO-DD HH:MI:SS");
"     Date last modified: ";; $bb ;


if missrv(make_vars1,0)==0;
  call grbl2_error(  "Error: you did not specify a variable list  (or specify that varnames are on first line) ",1);
   stop;
endif;

if not(scalmiss(make_miss)) ;
  "  Missing value flag: " ;; format /ld 10,1 ; make_miss ;
endif;

if make_1row==1 ; 
  "Each row of input file is used to create a row in the output file ";
endif;

if make_excel==1 ;		@ excel input file @
  xdata=xlsreadm(make_in,make_excel_1,make_excel_s,"");  @ read whole thing at once, then pull out lines @
  "# of rows read from excel file: ";; format /ld 8,0 ; rows(xdata) ; 
endif;


@ open up text file @
if make_excel ne 1 ;
  fin=fopen(make_in,"r");
  if fin==0 ;
        call grbl2_error("Error opening ascii data file: " $+ make_in,1) ;
     stop;
  endif;
endif;

if make_vars1==2 and make_excel ne 1 ;		@ read from rth line of text file @
  idd=parse(make_vars1_r,":");
  if rows(idd)== 1 ;
      iat1=stof(idd[1]);
      iat2=iat1 ;
  else;
      iat1=stof(idd[1]);
      iat2=stof(idd[2]);
  endif;
  alines=upper(fgetsa(fin,iat2));
  aline="" ;
  for mm (iat1,iat2,1) ;
     aline=aline $+ " " $+ alines[mm];
  endfor;

  foo=vals(aline); foo2=substute(foo,foo.==9,32);
  make_vars=upper(chrs(foo2));
  tmp={};
  do while 1==1 ;
            if make_vars$==""  ; break ;endif;
	    {aword,make_vars}=token(make_vars) ;
            tmp=tmp|aword;
  endo;
  make_vars=tmp ;
endif ;

if make_vars1==2 and make_excel eq 1 ;		@ read from rth line of excel file @
  foo=make_vars1_r ;
  if make_vars1_r$=="1" ; foo="A1:FF1"; endif;
  make_vars=xlsreadm(make_in,foo,make_excel_s,"");  @ read whole thing at once, then pull out lines @
  make_vars=vec(make_vars');
  make_vars=upper(packr(make_vars));
endif ;

for i (1,rows(make_vars),1);
  make_vars[i]=g_strip(upper(make_vars[i]));
endfor;

foo=g_showvec(make_vars,6,16,"  Variables ");

call g_pause(1,0);

numvars=rows(make_vars);
dat0=miss(zeros(numvars,1),0);    @ row template @
ncol=0; totbytes=0; totrows=0; wrows=0;
dat={};

create fhh=^make_out with ^make_vars,0,4;
if fhh<0 ;
  call grbl2_error("ERROR: unable to open file for outut: " $+ make_out,1) ;
   stop;
endif;

ir=csrlin ; ic=csrcol ;
ith=0;
iat=1; iat2=0;
xrows=0;
if make_excel==1 ;
   if (make_vars1==2)  ;   @ skip first line (names line) @
       xdata=xdata[2:rows(xdata),.];
   endif;
   xrows=rows(xdata); 
endif;

didbreak=0;

do while 1==1 ;

 if make_excel ne 1 ;
    if eof(fin) ; break ; endif;
    alines=upper(fgetsa(fin,100));
 else ;
     if iat>xrows; break ; endif;
     iat2=minc((iat+99)|xrows);
     alines=xdata[iat:iat2,.];
     iat=iat2+1 ;
 endif;
 totrows=totrows+rows(alines);

 for jj (1,rows(alines),1);
   
   if make_excel ne 1 ;
      aline=alines[jj];
      if vals(aline)==10 ; 
         if jj ne rows(alines) ; continue  ; endif ;
         tmp=miss(1,1);
      else;
         totbytes=totbytes+strlen(aline);
         tmp=stof(aline);
      endif;  @ skip blank lines @
   else ;
      tmp=alines[jj,.]';
   endif;
   if make_1row==1 ;	@  row per row specified? @
      if not(scalmiss(tmp)) ;       @ hack to deal with blank lines ending file @
	    if rows(tmp)>numvars ;    tmp=tmp[1:numvars];   endif;  @ trim excess values @
        if rows(tmp)< numvars ; tmp1=dat0; tmp1[1:rows(tmp)]=tmp ; tmp=tmp1 ; endif; @ pad with missings @
        dat=dat|(tmp') ;
      endif;
      if jj==rows(alines) ; 
          if not(scalmiss(make_miss)) ; dat=miss(dat,make_miss); endif;
          foo=writer(fhh,dat); wrows=wrows+rows(dat); dat={}; 
      endif;
      continue ;
   endif;

   if ncol==0 and rows(tmp)==numvars ; 
        if not(scalmiss(make_miss)) ; tmp=miss(tmp,make_miss); endif;
        foo=writer(fhh,tmp'); dat={} ;
        wrows=wrows+1 ;
   else ;
      for i (1,rows(tmp),1);
         ncol=ncol+1 ;
         if ncol==1 ; dat=dat0; endif;  @ beginning new row @
         dat[ncol]=tmp[i];
         if ncol== numvars ;  
            if not(scalmiss(make_miss)) ; dat=miss(dat,make_miss); endif;
            foo=writer(fhh,dat'); wrows=wrows+1 ;
	        ncol=0 ;  
         endif;		@ end of row @
      endfor;		@ elements in tmp @
    endif;		@ not simple row append @

 endfor;		@ aline @
 ith=ith+1 ;

 if ith==10 ;
   scroll ir|ic|Ir|80|0|0; 
   locate ir,ic ;  " ...  # of rows read: " ;; format /ld 10, 0 ; totrows ;;
   if make_excel ne 1 ;    "  % complete= " ;;format /ld 5,2; (100*totbytes/finfo[1,8]);;  endif;
  
   foo=g_pause2(" ... hit ESC to stop ... ",0.0001);
   if foo==27 ;" ESC hit -- no more data will be read " ;didbreak=1; break ; endif;
   ith=ith+1;
   if ith>10 ; ith=0 ; endif;
 endif;

endo;	 @ eof @
ith= close(fhh);
?; ?;      



output on ;
"-----------------------------------------";
" New GAUSS dataset created. ";
if make_excel==1 ;
  " : Excel input file= " ;; $make_in ;
else ;
  " : Ascii (text) input file= " ;; $make_in ;
endif;
" : Newly created gauss dataset= " ;; $make_out ;
if didbreak== 1 ; 
  "Note: data read/write stopped early (about ";;
  format /ld 6,2 ; (100*totbytes/finfo[1,8]);; 
   "% of input file was used)";
endif;

format /ld 8,0; 

"    Number of rows read= " ;; totrows ;
"    Number of rows written= " ;; wrows ;

foo=g_showvec(make_vars,6,16,"  Variables: ");
?;

output off; 

if make_excel ne 1 ; foo=close(fin);  endif;


call g_pause2("Hit any key to continue ",2);?;

closeall ;
return ;


