summaryrefslogtreecommitdiff
path: root/support/word2x/strip.cc.orig
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/word2x/strip.cc.orig
Initial commit
Diffstat (limited to 'support/word2x/strip.cc.orig')
-rw-r--r--support/word2x/strip.cc.orig522
1 files changed, 522 insertions, 0 deletions
diff --git a/support/word2x/strip.cc.orig b/support/word2x/strip.cc.orig
new file mode 100644
index 0000000000..5d0af50e56
--- /dev/null
+++ b/support/word2x/strip.cc.orig
@@ -0,0 +1,522 @@
+/* $Id: junk_filter.cc,v 1.40 1998/06/14 06:25:36 dps Exp $ */
+
+/* This code filters out almost all the "junk" in word documents and
+ * is useful for extracting the text from word documents. I think this
+ * program produces better signal to noise that catdoc. */
+
+/* #include "config.h" */
+#define HAVE_STRING_H 1
+#define HAVE_STRINGS_H 1
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif /* HAVE_STRING_H */
+#ifdef HAVE_STRINGS_H
+#include <strings.h>
+#endif /* HAVE_STRINGS_H */
+#include <ctype.h>
+#include <stream.h>
+#include <iostream.h>
+#include <fstream.h>
+#include "strip.h"
+#include "tune.h"
+
+/* Note: This is a two stage filter that outputs 0 or 1 characters
+ per input character and thus can use the same buffer safely. */
+
+/* Read more */
+int word_junk_filter::__fill(const char *b, int n, int sp, int offset)
+{
+ int end, nr;
+
+ if (n>0)
+ memmove(buf+offset, b, n);
+ offset+=n+sp;
+ if (offset>=JUNK_FILT_BUFSIZE)
+ {
+ cerr<<"junk_filter::__fill: start offset "<<offset<<" >= buffer size"
+ "(fatal)\n";
+ exit(1);
+ }
+ is->read(buf+offset, JUNK_FILT_BUFSIZE-offset);
+ nr=is->gcount();
+ pos+=nr;
+ end=nr+n;
+ return end;
+}
+
+
+/* Junk filtering. Buffer size must be >=RESUME_CHARS */
+int word_junk_filter::filter_junk(const char *b, int ns)
+{
+ int i, d, n, top_run;
+
+ if ((n=__fill(b, ns, junk_end_usage, 1))==0)
+ return 0;
+ if (junk_end_usage>0)
+ memcpy(buf+ns+1, junk_end_buf, junk_end_usage);
+ if (init==INIT_SKIPPED)
+ {
+ init=INIT_DONE; // Finished init
+ ns=0; // Process all the data
+ text_size=0; // Text size is zero
+ }
+
+#ifdef DEBUG_FILTER_FILL
+ printf("\n---refill ns=%d junk_end_usage=%d\n", ns, junk_end_usage);
+ printf("---end=%d mode=%d\n", n, mode);
+#endif
+
+ /* Correctness of this loop depends on d<=i at the top of the
+ loop. The claims * by inductioon about the fake variable __adj__ suffice
+ to prove this. All the relevant commments are prefixed by CS */
+
+ top_run=0;
+ /*** CS: __adj__=1 ****/
+ for (i=ns+1, d=ns; i<n; i++) // Ignore already processed data pushed back
+ {
+ /*** CS: i==d+junk_end_usage+__adj__ and __adj__>=0 here by induction ***/
+ /*** CS: if mode==SKIP_JUNK or SKIP_JUNK_WASPRN then __adj__>=1 ***/
+#ifdef DEBUG_FILTER
+ printf("mode %d, char[%i] (t=%d) %c, asc=%d\n", mode, i, n,
+ buf[i], junk_end_usage);
+#endif
+ switch(mode)
+ {
+ case NORMAL:
+ /* Normal mode. OK becasue change in d at most 1 and junk_end_usage=0 */
+ if (buf[i]==0)
+ {
+ junk_end_usage=0;
+ junk_size=0;
+ mode=SKIP_JUNK;
+ break;
+ }
+ if (buf[i] & 0x80)
+ {
+ /*** CS: by induction junk_end_usage==top_run ***/
+ junk_end_buf[junk_end_usage++]=buf[i]; // Save character
+ top_run++; // Increment top run counter
+ /*** CS: __adj__++ ***/
+ if (top_run==tune.max_top_run)
+ {
+ junk_size=top_run; // Top bit characters are junk
+ junk_end_usage=0; // Throw away characters
+ mode=SKIP_JUNK; // End junk skipping mode
+ /*** CS: __adj__ += top_run ***/
+ break;
+ }
+ break; // Exit switch
+ }
+ /* Top bit not set */
+ if (top_run)
+ {
+ memcpy(buf+d, junk_end_buf, junk_end_usage); // OK by ind hyp.
+ d+=junk_end_usage;
+ junk_end_usage=0; // Ind. hyp now true
+ top_run=0; // Reset top_run counter.
+ }
+ /* Add character */
+ buf[d++]=buf[i];
+ text_size++;
+ if (unicode!=__UNI_NO)
+ mode=UNICODE_Z;
+ break;
+
+ case UNICODE_Z:
+ /* unicode zero mode. OK because change in d is 0 and junk_end_usage=0 */
+ if (buf[i]==0)
+ {
+ mode=NORMAL; // 0 => Back to printable character mode
+ /*** CS: __adj__++ ***/
+ }
+ else
+ {
+#ifdef UNICODE_MARKERS
+ printf("\n*** Mode %d: %d Got %d expected 0\n", mode,
+ UNICODE_Z, buf[i]);
+#endif
+ junk_end_usage=0;
+ junk_size=0;
+ mode=SKIP_JUNK;
+ /*** CS: __adj__++ ***/
+ }
+ break;
+
+ case SKIP_JUNK_WASPRN:
+ /* Skip junk was printable, added for unicode documents */
+ /* OK because junk_end_usage chages by at most 1 and d stays fixed */
+ if ((++junk_size)==tune.max_junk && text_size>tune.min_text)
+ {
+#ifdef DEBUG_SINK_START
+ cerr<<"Enter sink mode\n";
+#endif /* DEBUG_SINK_START */
+ /*** CS: __adj__++ ***/
+ mode=SINK_JUNK; // Sink junk
+ break;
+ }
+ if (tune.options.unicode_aggresive)
+ {
+ /* Agresive unicode mode: insist on the MSBs appearing */
+ if (unicode!=__UNI_NO)
+ {
+ if (buf[i]!=0)
+ {
+ /*** CS: __adj__+=junk_end_usage ***/
+ junk_end_usage=0;
+ }
+ mode=SKIP_JUNK;
+ /*** CS: __adj__++ ***/
+ break;
+ }
+ }
+ else
+ {
+ /* Non-aggersive unicode mode: simply skip the MSB */
+ if (buf[i]==0 && unicode!=__UNI_NO)
+ {
+ /*** CS: __adj__++ ***/
+ break; // Accept zeros
+ }
+ }
+ /* Fall thru */
+
+ case SKIP_JUNK:
+ /* Junk skipping mode */
+ if ((++junk_size)==tune.max_junk && text_size>tune.min_text)
+ {
+#ifdef DEBUG_SINK_START
+ cerr<<"Enter sink mode\n";
+#endif /* DEBUG_SINK_START */
+ mode=SINK_JUNK; // Sink junk
+ /*** CS: __adj__++ ***/
+ break;
+ }
+ if (isprint(buf[i]) || buf[i]=='\r' || buf[i]=='\n'
+ || buf[i]=='\t')
+ {
+#ifdef DEBUG_RESTART
+ printf("---Accepted *%c*, usage=%d\n", buf[i], junk_end_usage);
+#endif
+ junk_end_buf[junk_end_usage++]=buf[i]; // Save character
+ if (unicode!=__UNI_NO)
+ mode=SKIP_JUNK_WASPRN; // For unicode documents
+ }
+ else
+ {
+ /* Reset junk buffer usage counter */
+ /*** CS: __adj__+=junk_end_usage+1 ***/
+#ifdef DEBUG_RESTART
+ if (junk_end_usage!=0)
+ printf("---Rejected %d, usage=%d\n", buf[i], junk_end_usage);
+#endif
+ junk_end_usage=0;
+ break;
+ }
+ if (junk_end_usage<tune.resume_chars)
+ break;
+ /* Turn nomral mode back on */
+#ifdef DEBUG_RESTART
+ printf("---back normal mode count=%d\n", junk_size);
+#endif
+ /* The __adj__ analysis is *vital* here; if __Adj__=0 then d>i occurs */
+ /* however forutnately this is impossible by the claims that __adj__ */
+ /* and indcution proves. */
+
+ /**** CS: __adj__>=1 here by indcution. Thus i>=d+junk_end_usage+1 ****/
+ mode=(unicode==__UNI_NO) ? NORMAL : UNICODE_Z;
+ buf[d]=0x7f; // Mark suspect position
+ memcpy(buf+d+1, junk_end_buf, junk_end_usage); // Nothing clobbered by [1]
+ d+=junk_end_usage+1; // Advance d
+ junk_end_usage=0; // Clear junk end buffer
+ /*** CS: __adj__-- ***/
+ break;
+
+ case SINK_JUNK:
+ /*** CS: __adj__++ ***/
+ break;
+
+ default:
+ cerr<<"word_junk_filter::filter_junk: Invalid mode "<<mode
+ <<" (fatal)\n";
+ exit(1);
+ }
+ }
+ return d;
+}
+
+int word_junk_filter::underflow(void)
+{
+ int s,d,last;
+ int end;
+
+ do // While no characters
+ {
+ /* Fill the buffer */
+ if (init==INIT_NONE)
+ {
+ if (is!=NULL)
+ {
+ init=INIT_SKIPPED;
+ ns=this->skip_to_start();
+ end=this->filter_junk(buf, ns);
+ }
+ else
+ return EOF; // If no input stream return EOF
+ }
+ else
+ {
+ end=this->filter_junk(&save, ns);
+ ns=0;
+ }
+
+ /* Check for EOF */
+ if (end==0 && is->eof())
+ return EOF;
+
+ ns=0;
+ /* Collapse multi-character sequeneces */
+ for (s=0, d=0, last=-1; s<end; s++)
+ {
+ switch(buf[s])
+ {
+ case '\n':
+ if (last=='\r')
+ continue; // \n \r to \r
+ break;
+
+
+ case '\r':
+ if (last=='\n')
+ d--; // \r \n to \r
+ break;
+
+
+ case 0x07:
+ if (last=='\r')
+ d--; // \n <TAB SEP> to <TAB SEP>
+ break;
+
+ default:
+ break;
+ }
+ last=buf[d]=buf[s];
+ d++;
+ }
+
+ if (end!=1 && (buf[d-1]=='\r' || buf[d-1]=='\n'))
+ {
+ d--;
+ ns=1;
+ save=buf[d];
+ }
+ } while (d==0);
+ setg(buf, buf, buf+d);
+ return(buf[0] & 0xff);
+}
+
+
+/* Skip to start */
+int word_junk_filter::skip_to_start(void)
+{
+ int zc,ffc,c,n,i,rej,limit=-1;
+ char back[2*ST_ASC_VCHARS_UNICODE+ST_REJ_LIMIT+4];
+ int bp;
+ long pos=0;
+
+ enum { FIND_BEGIN, SKIP_ONE, FIND_ASC, FIND_ASC_WASASC, DONE } mode;
+ zc=0; ffc=0; i=0; n=0; bp=0; rej=-1;
+ mode=FIND_BEGIN;
+ while (mode!=DONE)
+ {
+ /* If need a refill */
+ if (i==n)
+ {
+ /* Fill buffer */
+ n=this->__fill(back,bp,0,0);
+ if (n==0)
+ return 0;
+ pos+=n-bp;
+#ifdef DEBUG_SKIP_FILL
+ printf("---skip--- refill bp=%x, pos=%x\n", bp, pos);
+#endif
+ /* Ignore first saved character */
+ if (mode==FIND_ASC)
+ mode=SKIP_ONE;
+ i=0;
+ bp=0;
+ }
+
+#ifdef DEBUG_SKIP
+ printf("char %c mode %d, c=%d, pos=%x, i=%d l=%d, rej=%d\n", buf[i],
+ mode, c, pos++, i, limit, rej);
+#endif
+ switch(mode)
+ {
+ case FIND_BEGIN:
+ switch(buf[i])
+ {
+ case 0:
+ zc++;
+ ffc=0;
+ break;
+
+ case ((char) 0xff):
+ ffc++;
+ zc=0;
+ break;
+
+ default:
+ if (zc>=tune.st_min_zeros ||
+ (ffc>=tune.st_min_ff && tune.options.ff_intro))
+ {
+ unicode=__UNI_PROBE;
+ back[bp++]=buf[i];
+ c=0;
+ rej=0;
+ mode=FIND_ASC_WASASC; // Ignore this character deliberately
+ }
+ else
+ {
+ zc=0;
+ ffc=0;
+ }
+ break;
+ }
+ break;
+
+ case SKIP_ONE:
+ back[bp++]=buf[i];
+ c=0;
+ rej=0;
+ mode=FIND_ASC_WASASC;
+ break;
+
+ case FIND_ASC_WASASC: // unicode support
+ if (buf[i]==0 && unicode!=__UNI_NO)
+ {
+ if (unicode==__UNI_PROBE)
+ {
+ limit=tune.unicode_st; // Start characters needed
+ /* FIXME: Anyone got a better method of doing this? */
+ if ((pos+i) % 2==0)
+ {
+ unicode=__UNI_YES_BE; // 0 is first (even) bu#yte
+ }
+ else
+ {
+ unicode=__UNI_YES_LE; // 0 is second (odd) byte
+ }
+ back[bp++]=buf[i];
+ mode=FIND_ASC; // Inist next character not zero
+#ifdef DEBUG_SKIP_UNI
+ cerr<<"Characters are unicode "
+ <<((unicode==__UNI_YES_LE) ? "litle" : "big")
+ <<" endian\n";
+#endif
+ }
+ else
+ back[bp++]=buf[i];
+ c++;
+ break;
+ }
+ else if (unicode==__UNI_PROBE)
+ {
+#ifdef DEBUG_SKIP_UNI
+ if (unicode==__UNI_PROBE)
+ cerr<<"Characters are not unicode\n";
+#endif
+ limit=tune.non_unicode_st; // Start characters needed
+ unicode=__UNI_NO; // Non unicode
+ mode=FIND_ASC; // Finding printable mode
+ }
+ /* fall thru */
+
+ case FIND_ASC:
+ if (isprint(buf[i]) || buf[i]=='\r' || buf[i]=='\n'
+ || buf[i]=='\t') // If is printable
+ {
+ back[bp++]=buf[i]; // Store character
+ c++; // Increase count
+ if (unicode!=__UNI_NO)
+ mode=FIND_ASC_WASASC; // Unicode hack...
+ }
+ else
+ {
+ back[bp++]=buf[i]; // Store caharcetr
+ if (buf[i]!=0)
+ rej++; // Adjust reject count
+ if (buf[i]==0 || rej>tune.st_rej_limit) // If 0 or reject limit
+ {
+ bp=0; // Reset buffer pointer
+ unicode=__UNI_PROBE; // Set unicode status to unknonw
+ ffc=(buf[i]== (char) 0xff) ? 1 : 0; // reset ff count
+ zc=(buf[i]==0) ? 1 : 0; // reset zeros count
+ mode=FIND_BEGIN ; // Find leading characters mode
+ }
+ }
+ if (limit==-1) // Bug catcher
+ {
+ cerr<<"Bug: limit still -1 at point #1\n";
+ exit(2);
+ }
+ if (c==limit) // IF found enough
+ {
+#ifdef DEBUG_SKIP
+ printf("***** Enter exit phase ****\n");
+#endif
+ mode=DONE; // Finished skipping
+ }
+ break;
+
+ case DONE:
+ cerr<<"skip_to_start: loop called in done mode\n";
+ break;
+
+ default:
+ cerr<<"skip_to_start: Mode invalid\n";
+ break;
+ }
+ i++;
+ }
+#ifdef DEBUG_SKIP_MOVE
+ printf("%c%c%c\n", buf[i-bp], buf[i-bp+1], buf[i-bp+2]);
+#endif
+ memmove(buf, buf+i-bp, n-i+bp); // Move memory
+ return(n-i+bp); // Return count
+}
+
+
+/* Option handling stuff for tuning this module. Note that only long options
+ * are supported. If you lack long options these features are not avialable. */
+/* set default tuning */
+void word_junk_filter::set_dfl_tuning(void)
+{
+#ifdef AGGRESIVE_UNICODE
+ tune.options.unicode_aggresive=AGGRESIVE_UNICODE; // Aggresive unicode mode
+#else
+ tune.options.unicode_aggresive=1; // Aggresive unicode mode
+#endif /* AGGRESIVE_UNICODE */
+ tune.resume_chars=RESUME_CHARS; // Resume characters
+ tune.st_min_zeros=ST_MIN_ZEROS; // Minimum zeros
+ tune.st_rej_limit=ST_REJ_LIMIT; // Reject chars tolerance
+ tune.st_min_ff=ST_MIN_FF; // Minimum ff's (alternative to 0s)
+ tune.options.ff_intro=ALLOW_FF_LEADIN; // Enable leading ff's
+ tune.non_unicode_st=ST_ASC_VCHARS; // ok characters leadin
+ tune.unicode_st=ST_ASC_VCHARS_UNICODE; // ok unicode characters
+ tune.min_text=SINK_MIN_TEXT; // text required before sink mode
+ tune.max_junk=SINK_JUNK_CNT; // junk required for sink mode
+ tune.max_top_run=MAX_TOP_RUN; // Max run length of top bits set
+}
+
+int word_junk_filter::do_reset(void)
+{
+ pos=0;
+ init=INIT_NONE;
+ mode=NORMAL;
+ junk_end_usage=0;
+ ns=0;
+ return 1; // Success
+}
+