summaryrefslogtreecommitdiff
path: root/support/word2x/strip.h
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/word2x/strip.h
Initial commit
Diffstat (limited to 'support/word2x/strip.h')
-rw-r--r--support/word2x/strip.h135
1 files changed, 135 insertions, 0 deletions
diff --git a/support/word2x/strip.h b/support/word2x/strip.h
new file mode 100644
index 0000000000..57da610b64
--- /dev/null
+++ b/support/word2x/strip.h
@@ -0,0 +1,135 @@
+/* $Id: junk_filter.h,v 1.16 1998/06/14 05:17:25 dps Exp $ */
+
+#ifndef __JUNK_FILTER_H__
+#define __JUNK_FILTER_H__
+#include <iostream.h>
+#include <fstream.h>
+#include <stdlib.h>
+#include "tune.h"
+
+class word_junk_filter: public streambuf
+{
+private:
+ enum { INIT_NONE, INIT_SKIPPED, INIT_DONE } init; /* Init stage */
+ istream *is; /* Data source */
+ enum
+ {
+ __UNI_PROBE, /* Unknown, probe */
+ __UNI_NO, /* No */
+ __UNI_YES_LE, /* Yes, little endian */
+ __UNI_YES_BE /* Yes, big endian */
+ } unicode; /* Unicode flag */
+
+ char junk_end_buf[RESUME_CHARS+1]; /* Junk end characters */
+ int junk_end_usage; /* Usage of junk end buffer */
+ int text_size; /* Text characters retreived */
+ int junk_size; /* Amount of junk skipped */
+
+ char buf[JUNK_FILT_BUFSIZE+2];
+ enum {
+ NORMAL, UNICODE_Z, /* Unicode, Unicode zero expected */
+ SKIP_JUNK, SKIP_JUNK_WASPRN, /* Skip junk, Skip junk was printable */
+ SINK_JUNK, /* Like skip junk with more restart */
+ } mode;
+ char save;
+ int ns;
+
+ /* Tunable stuff */
+ struct
+ {
+ int st_min_zeros; // Minimum leading zeros
+ int st_min_ff; // Minimum leadingg ff characters
+ int st_rej_limit; // Non-printing characters tolerated
+ int non_unicode_st; // Not unicode start treshold
+ int unicode_st; // Unicode start treshold
+ int max_junk; // Junk required to trigger SINK_JUNK mode
+ int min_text; // Text required to trigger SINK_JUNK mode
+ int max_top_run; // Max top bit set run
+ struct
+ {
+ unsigned unicode_aggresive: 1; // Aggresive unicode mod
+ unsigned ff_intro:1; // Allow ff leadin characters
+ } options;
+ int resume_chars; // Good characters need to stop junk skipping
+ } tune;
+ void set_dfl_tuning(void); // Set default tuning
+
+ int __fill(const char *, int, int, int);
+ int filter_junk(const char *, int);
+ int skip_to_start(void);
+ long pos;
+
+
+
+public:
+ inline int overflow(int ch)
+ {
+ ch=ch;
+ return EOF; /* ZXZ */
+ }
+ inline int sync(void)
+ {
+ return 0;
+ }
+
+ int underflow(void);
+ int do_reset(void);
+ inline int __uflow(void) { return this->underflow(); }
+
+ inline void open(istream *s)
+ {
+ if (is!=NULL)
+ delete(is);
+ is=s;
+ pos=0;
+ init=INIT_NONE;
+ mode=NORMAL;
+ junk_end_usage=0;
+ ns=0;
+ }
+
+ inline word_junk_filter(void)
+ {
+ set_dfl_tuning(); // Set default tuning
+ is=NULL;
+ init=INIT_NONE;
+ }
+
+ inline word_junk_filter(istream *s)
+ {
+ set_dfl_tuning(); // Set default tuning
+ pos=0;
+ init=INIT_NONE;
+ junk_end_usage=0;
+ is=s;
+ mode=NORMAL;
+ ns=0;
+ }
+
+ inline word_junk_filter(const char *f)
+ {
+ ifstream *s;
+
+ set_dfl_tuning(); // Set default tuning
+ s=new(ifstream)(f);
+ if (!(*s))
+ {
+ cerr<<"Can not open "<<f<<" (fatal)\n";
+ exit(1);
+ }
+ is=s;
+ pos=0;
+ init=INIT_NONE;
+ junk_end_usage=0;
+ mode=NORMAL;
+ ns=0;
+ }
+
+ inline ~word_junk_filter(void)
+ {
+ if (is!=NULL)
+ delete(is);
+ }
+};
+
+#endif /* __JUNK_FILTER_H__ */