% First a special index style for makeindex % \begin{filecontents}{stex.ist} actual '=' quote '!' level '>' preamble "\n \\begin{theindex} \n \\makeatletter\\scan@allowedfalse\n" postamble "\n\n \\end{theindex}\n" item_x1 "\\efill \n \\subitem " item_x2 "\\efill \n \\subsubitem " delim_0 "\\pfill " delim_1 "\\pfill " delim_2 "\\pfill " % The next lines will produce some warnings when % running Makeindex as they try to cover two different % versions of the program: lethead_prefix "{\\bfseries\\hfil " lethead_suffix "\\hfil}\\nopagebreak\n" lethead_flag 1 heading_prefix "{\\bfseries\\hfil " heading_suffix "\\hfil}\\nopagebreak\n" headings_flag 1 % and just for source2e: % Remove R so I is treated in sequence I J K not I II III page_precedence "rnaA" \end{filecontents} \documentclass{ltxdoc} \usepackage{stex,cmathml,cmathmlx} \usepackage{lstdoc,textcomp,url,array,stex,float,chapterbib,amsfonts} \usepackage[show]{ed} \usepackage{hyperref} \makeindex \floatstyle{boxed} \newfloat{exfig}{thp}{lop} \floatname{exfig}{Example} \def\scsys#1{{{\sc #1}}\index{#1@{\sc #1}}} \def\stex{{\raisebox{-.5ex}S\kern-.5ex\TeX}} \def\sTeX{\stex} \def\xslt{{\scsys{xslt}}} \def\xml{\scsys{Xml}} \def\mathml{\scsys{MathML}} \def\omdoc{\scsys{OMDoc}} \def\openmath{\scsys{OpenMath}} \def\latexml{\scsys{LaTeXML}} \def\connexions{\scsys{Connexions}} \def\latexml{\scsys{LaTeXML}} \def\perl{\scsys{Perl}} \def\cmathml{Content-{\sc MathML}\index{Content {\sc MathML}}\index{MathML@{\sc MathML}!content}} \def\activemath{\scsys{ActiveMath}} \def\twin#1#2{\index{#1!#2}\index{#2!#1}} \def\twintoo#1#2{{#1 #2}\twin{#1}{#2}} \def\atwin#1#2#3{\index{#1!#2!#3}\index{#3!#2 (#1)}} \def\atwintoo#1#2#3{{#1 #2 #3}\atwin{#1}{#2}{#3}} % these macros are used in the short descriptions \def\connexions{\scshape{Connexions}} \def\cnxlatex{CNX\LaTeX} \def\cnxml{\scshape{CNXml}} %%%%%%%%%%%%%% begin copied from source2e.tex \listfiles % Do not index some TeX primitives, and some common plain TeX commands. \DoNotIndex{\def,\long,\edef,\xdef,\gdef,\let,\global} \DoNotIndex{\if,\ifnum,\ifdim,\ifcat,\ifmmode,\ifvmode,\ifhmode,% \iftrue,\iffalse,\ifvoid,\ifx,\ifeof,\ifcase,\else,\or,\fi} \DoNotIndex{\box,\copy,\setbox,\unvbox,\unhbox,\hbox,% \vbox,\vtop,\vcenter} \DoNotIndex{\@empty,\immediate,\write} \DoNotIndex{\egroup,\bgroup,\expandafter,\begingroup,\endgroup} \DoNotIndex{\divide,\advance,\multiply,\count,\dimen} \DoNotIndex{\relax,\space,\string} \DoNotIndex{\csname,\endcsname,\@spaces,\openin,\openout,% \closein,\closeout} \DoNotIndex{\catcode,\endinput} \DoNotIndex{\jobname,\message,\read,\the,\m@ne,\noexpand} \DoNotIndex{\hsize,\vsize,\hskip,\vskip,\kern,\hfil,\hfill,\hss} \DoNotIndex{\m@ne,\z@,\z@skip,\@ne,\tw@,\p@} \DoNotIndex{\dp,\wd,\ht,\vss,\unskip} % Set up the Index and Change History to use \part \IndexPrologue{\part*{Index}% \markboth{Index}{Index}% \addcontentsline{toc}{part}{Index}% The italic numbers denote the pages where the corresponding entry is described, numbers underlined point to the definition, all others indicate the places where it is used.} \GlossaryPrologue{\part*{Change History}% % Allow control names to be hyphenated here... {\GlossaryParms\ttfamily\hyphenchar\font=`\-}% \markboth{Change History}{Change History}% \addcontentsline{toc}{part}{Change History}} % The standard \changes command modified slightly to better cope with % this multiple file document. \makeatletter \def\changes@#1#2#3{% \let\protect\@unexpandable@protect \edef\@tempa{\noexpand\glossary{#2\space\currentfile\space#1\levelchar \ifx\saved@macroname\@empty \space \actualchar \generalname \else \expandafter\@gobble \saved@macroname \actualchar \string\verb\quotechar*% \verbatimchar\saved@macroname \verbatimchar \fi :\levelchar #3}}% \@tempa\endgroup\@esphack} \makeatother % Produce a Change Log and (2 column) Index. \RecordChanges \CodelineIndex \EnableCrossrefs \OnlyDescription \setcounter{IndexColumns}{2} %%%%%%%%%%%%%% end copied from source2e.tex %\includeonly{} \title{{\stex}: Semantic Markup in {\TeX/\LaTeX}} \author{Michael Kohlhase\\ Jacobs University, Bremen\\ \url{http://kwarc.info/kohlhase}} \begin{document} \pagenumbering{roman} \maketitle \renewcommand\maketitle{} \begin{abstract} We present a collection of {\TeX} macro packages that allow to markup {\TeX/\LaTeX} documents semantically without leaving the document format, essentially turning {\TeX/\LaTeX} into a document format for mathematical knowledge management (MKM). \end{abstract} \newsavebox{\forgetbox} \renewenvironment{abstract}{\begin{lrbox}{\forgetbox}}{\end{lrbox}} \newpage \setcounter{tocdepth}{1}\tableofcontents \clearpage \pagenumbering{arabic} \part{The sTeX Collection} \section{Introduction} The last few years have seen the emergence of various content-oriented {\xml}-based, content-oriented markup languages for mathematics on the web, e.g. {\openmath}~\cite{BusCapCar:2oms04}, {\cmathml}~\cite{CarIon:MathML03}, or our own {\omdoc}~\cite{Kohlhase:omfmd05}. These representation languages for mathematics, that make the structure of the mathematical knowledge in a document explicit enough that machines can operate on it. Other examples of content-oriented formats for mathematics include the various logic-based languages found in automated reasoning tools (see~\cite{RobVor:hoar01} for an overview), program specification languages (see e.g.~\cite{Bergstra:as89}). The promise if these content-oriented approaches is that various tasks involved in ``doing mathematics'' (e.g. search, navigation, cross-referencing, quality control, user-adaptive presentation, proving, simulation) can be machine-supported, and thus the working mathematician is relieved to do what humans can still do infinitely better than machines: The creative part of mathematics --- inventing interesting mathematical objects, conjecturing about their properties and coming up with creative ideas for proving these conjectures. However, before these promises can be delivered upon (there is even a conference series~\cite{MKM-IG-Meetings:web} studying ``Mathematical Knowledge Management (MKM)''), large bodies of mathematical knowledge have to be converted into content form. Even though {\mathml} is viewed by most as the coming standard for representing mathematics on the web and in scientific publications, it has not not fully taken off in practice. One of the reasons for that may be that the technical communities that need high-quality methods for publishing mathematics already have an established method which yields excellent results: the {\TeX/\LaTeX} system: and a large part of mathematical knowledge is prepared in the form of {\TeX}/{\LaTeX} documents. {\TeX}~\cite{Knuth:ttb84} is a document presentation format that combines complex page-description primitives with a powerful macro-expansion facility, which is utilized in {\LaTeX} (essentially a set of {\TeX} macro packages, see~\cite{Lamport:ladps94}) to achieve more content-oriented markup that can be adapted to particular tastes via specialized document styles. It is safe to say that {\LaTeX} largely restricts content markup to the document structure\footnote{supplying macros e.g. for sections, paragraphs, theorems, definitions, etc.}, and graphics, leaving the user with the presentational {\TeX} primitives for mathematical formulae. Therefore, even though {\LaTeX} goes a great step into the direction of an MKM format, it is not, as it lacks infrastructure for marking up the functional structure of formulae and mathematical statements, and their dependence on and contribution to the mathematical context. \subsection{The {\xml} vs. {\TeX/\LaTeX} Formats and Workflows} {\mathml} is an {\xml}-based markup format for mathematical formulae, it is standardized by the World Wide Web Consortium in {\cite{CarIon:MathML03}}, and is supported by the major browsers. The {\mathml} format comes in two integrated components: presentation {\mathml}\twin{presentation}{MathML} and content {\mathml}\twin{content}{MathML}. The former provides a comprehensive set of layout primitives for presenting the visual appearance of mathematical formulae, and the second one the functional/logical structure of the conveyed mathematical objects. For all practical concerns, presentation {\mathml} is equivalent to the math mode of {\TeX}. The text mode facilitates of {\TeX} (and the multitude of {\LaTeX} classes) are relegated to other {\xml} formats, which embed {\mathml}. The programming language constructs of {\TeX} (i.e. the macro definition facilities\footnote{We count the parser manipulation facilities of {\TeX}, e.g. category code changes into the programming facilities as well, these are of course impossible for {\mathml}, since it is bound to {\xml} syntax.}) are relegated to the {\xml} programming languages that can be used to develop language extensions. transformation language {\xslt}~\cite{Deach:exls99,Kay:xpr00} or proper {\xml}-enabled The {\xml}-based syntax and the separation of the presentational-, functional- and programming/extensibility concerns in {\mathml} has some distinct advantages over the integrated approach in {\TeX/\LaTeX} on the services side: {\mathml} gives us better \begin{itemize} \item integration with web-based publishing, \item accessibility to disabled persons, e.g. (well-written) {\mathml} contains enough structural information to supports screen readers. \item reusability, searchabiliby and integration with mathematical software systems (e.g. copy-and-paste to computer algebra systems), and \item validation and plausibility checking. \end{itemize} On the other hand, {\TeX/\LaTeX}/s adaptable syntax and tightly integrated programming features within has distinct advantages on the authoring side: \begin{itemize} \item The {\TeX/\LaTeX} syntax is much more compact than {\mathml} (see the difference in Figure~\ref{fig:mathml-sum} and Equation ~\ref{eq:cmathml-sum}), and if needed, the community develops {\LaTeX} packages that supply new functionality in with a succinct and intuitive syntax. \item The user can define ad-hoc abbreviations and bind them to new control sequences to structure the source code. \item The {\TeX/\LaTeX} community has a vast collection of language extensions and best practice examples for every conceivable publication purpose and an established and very active developer community that supports these. \item There is a host of software systems centered around the {\TeX/\LaTeX} language that make authoring content easier: many editors have special modes for {\LaTeX}, there are spelling/style/grammar checkers, transformers to other markup formats, etc. \end{itemize} In other words, the technical community is is heavily invested in the whole {\index*{workflow}}, and technical know-how about the format permeates the community. Since all of this would need to be re-established for a {\mathml}-based workflow, the technical community is slow to take up {\mathml} over {\TeX/\LaTeX}, even in light of the advantages detailed above. \subsection{A {\LaTeX}-based Workflow for {\xml}-based Mathematical Documents} An elegant way of sidestepping most of the problems inherent in transitioning from a {\LaTeX}-based to an {\xml}-based workflow is to combine both and take advantage of the respective advantages. The key ingredient in this approach is a system that can transform {\TeX\LaTeX} documents to their corresponding {\xml}-based counterparts. That way, {\xml}-documents can be authored and prototyped in the {\LaTeX} workflow, and transformed to {\xml} for publication and added-value services, combining the two workflows. There are various attempts to solve the {\TeX/\LaTeX} to {\xml} transformation problem; the most mature is probably Bruce Miller's {\latexml} system~\cite{Miller:latexml}. It consists of two parts: a re-implementation of the {\TeX} {\index*{analyzer}} with all of it's intricacies, and a extensible {\xml} emitter (the component that assembles the output of the parser). Since the {\LaTeX} style files are (ultimately) programmed in {\TeX}, the {\TeX} analyzer can handle all {\TeX} extensions, including all of {\LaTeX}. Thus the {\latexml} parser can handle all of {\TeX/\LaTeX}, if the emitter is extensible, which is guaranteed by the {\latexml} binding language: To transform a {\TeX/\LaTeX} document to a given {\xml} format, all {\TeX} extensions\footnote{i.e. all macros, environments, and syntax extensions used int the source document} must have ``{\latexml} bindings''\index{LaTeXML}{binding}, i.e. a directive to the {\latexml} emitter that specifies the target representation in {\xml}. \section{The Packages of the sTeX Collection}\label{sec:packages} In the following, we will shortly preview the packages and classes in the {\stex} collection. They all provide part of the solution of representing semantic structure in the {\TeX/\LaTeX} workflow. We will group them by the conceptual level they address\ednote{come up with a nice overview figure here!} \subsection{Content Markup of Mathematical Formulae in {\TeX/\LaTeX}} The first two packages are concerned basically with the math mode in {\TeX}, i.e. mathematical formulae. The underlying problem is that run-of-the-mill {\TeX/\LaTeX} only specifies the presentation (i.e. what formulae look like) and not their content (their functional structure). Unfortunately, there are no good methods (yet) to infer the latter from the former, but there are ways to get presentation from content. Consider for instance the following ``standard notations''\footnote{The first one is standard e.g. in Germany and the US, the third one in France, and the last one in Russia} for binomial coefficients: $\left(n\atop k\right)$, $_nC^k$, $\mathcal{C}^n_k$, and $\mathcal{C}^k_n$ all mean the same thing: $n!\over k!(n-k)!$. This shows that we cannot hope to reliably recover the functional structure (in our case the fact that the expression is constructed by applying the binomial function to the arguments $n$ and $k$) from the presentation alone. The solution to this problem is to dump the extra work on the author (after all she knows what she is talking about) and give them the chance to specify the intended structure. The markup infrastructure supplied by the {\stex} collection lets the author do this without changing\footnote{However, semantic annotation will make the author more aware of the functional structure of the document and thus may in fact entice the author to use presentation in a more consistent way than she would usually have.} the visual appearance, so that the {\LaTeX} workflow is not disrupted. . We speak of {\twintoo{semantic}{preloading}} for this process and call our collection of macro packages {\stex} (Semantic {\TeX}). For instance, we can now write \begin{equation}\label{eq:cmathml-sum} |\CSumlLimits{k}1\infty{\Cexp{x}k}| \qquad\hbox{instead of the usual}\qquad |\sum_{k=1}^\infty x^k| \end{equation} In the first form, we specify that you are applying a function (|CSumLimits| $\hat=$ Sum with Limits) to 4 arguments: ({\sl{i}}) the bound variable $k$ (that runs from) ({\sl{ii}}) the number 1 (to) ({\sl{iii}}) $\infty$ (to infinity summing the terms) ({\sl{iv}}) |\Cexp{x}k| (i.e. x to the power k). In the second form, we merely specify hat {\LaTeX} should draw a capital Sigma character ($\sigma$) with a lower index which is an equation $k=1$ and an upper index $\infty$. Then it should place next to it an $x$ with an upper index $k$. Of course human readers (that understand the math) can infer the content structure from this presentation, but the {\latexml} converter (who does not understand the math) cannot, but we want to have the content {\mathml} expression in Figure~\ref{fig:mathml-sum} \begin{exfig} \begin{verbatim} k 1 xk \end{verbatim}\vspace*{-.6cm} \caption{Content {\mathml} Form of $\sum_{k=1}^\infty x^k$}\label{fig:mathml-sum} \end{exfig} Obviously, a converter can infer this from the first {\LaTeX} structure with the help of the curly braces that indicate the argument structure, but not from the second (because it does not understand the math). The nice thing about the |cmathml| infrastructure is that you can still run {\LaTeX} over the first form and get the same formula in the DVI file that you would have gotten from running it over the second form. That means, if the author is prepared to write the mathematical formulae a little differently in her {\LaTeX} sources, then she can use them in {\xml} and {\LaTeX} at the same time. \subsubsection{{\tt{cmathml}}: Encoding Content {\mathml} in {\TeX/\LaTeX}} The |cmathml| package provides a set of macros that correspond to the K-14 fragment of mathematics (Kindergarten to undergraduate college level ($\hat=14^{th}$ grade)). We have already seen an example above in equation (\ref{eq:cmathml-sum}), where the content markup in {\TeX} corresponds to a content {\mathml}-expression (and can actually be translated to this by the {\latexml} system.) However, the content {\mathml} vocabulary is fixed in the {\mathml} specification and limited to the K-14 fragment; the notation of mathematics of course is much larger and extensible on the fly. \subsubsection{{\tt{presentation}}: Flexible Presentation for Semantic Macros} The |presentation| package supplies an infrastructure that allows to specify the presentation of semantic macros, including preference-based bracket elision. This allows to markup the functional structure of mathematical formulae without having to lose high-quality human-oriented presentation in {\LaTeX}. Moreover, the notation definitions can be used by MKM systems for added-value services, either directly from the {\sTeX} sources, or after translation. \subsection{Mathematical Statements} \subsubsection{{\tt{statements}}: Extending Content Macros for Mathematical Notation} This package provides semantic markup facilities for mathematical statements like Theorems, Lemmata, Axioms, Definitions, etc. in {\stex} files. This structure can be used by MKM systems for added-value services, either directly from the {\sTeX} sources, or after translation. \subsubsection{{\tt{sproof}}: Extending Content Macros for Mathematical Notation} This package supplies macros and environment that allow to annotate the structure of mathematical proofs in {\stex} files. This structure can be used by MKM systems for added-value services, either directly from the {\sTeX} sources, or after translation. \subsection{Context Markup for Mathematics} \subsubsection{{\tt{modules}}: Extending Content Macros for Mathematical Notation} This package supplies a definition mechanism for semantic macros and a non-standard scoping construct for them, which is oriented at the semantic depency relation rather than the document structure. This structure can be used by MKM systems for added-value services, either directly from the {\sTeX} sources, or after translation. \subsection{Mathematical Document Classes} \subsubsection{Connexions Modules} {\cnxlatex} is a collection of {\LaTeX} macros that allow to write {\connexions} modules without leaving the {\LaTeX} workflow. Modules are authored in {\cnxlatex} using only a text editor, transformed to PDF and proofread as usual. In particular, the {\LaTeX} workflow is independent of having access to the {\connexions} system, which makes {\cnxlatex} attractive for the initial version of single-author modules. For publication, {\cnxlatex} modules are transformed to {\cnxml} via the {\latexml} translator and can be uploaded to the {\connexions} system. \subsubsection{OMDoc Documents} The |omdoc| package provides an infrastructure that allows to markup {\omdoc} documents in {\LaTeX}. It provides |omdoc.cls|, a class with the and |omdocdoc.sty| \subsubsection{Slides and Presentations} We present a document class from which we can generate both course slides and course notes in a transparent way. Furthermore, we present a set of {\latexml} bindings for these, so that we can also generate {\omdoc}-based course materials, e.g. for inclusion in the {\activemath} system. \subsection{Conclusion}\label{sec:concl} The {\stex} collection provides a set of semantic macros that extends the familiar and time-tried {\LaTeX} workflow in academics until the last step of Internet publication of the material. For instance, a {\connexions} module can be authored and maintained in {\LaTeX} using a simple text editor, a process most academics in technical subjects are well familiar with. Only in a last publishing step (which is fully automatic) does it get transformed into the {\xml} world, which is unfamiliar to most academics. Thus, {\stex} can serve as a conceptual interface between the document author and MKM systems: Technically, the semantically preloaded {\LaTeX} documents are transformed into the (usually {\xml}-based) MKM representation formats, but conceptually, the ability to semantically annotate the source document is sufficient. The {\stex} macro packages have been validated together with a case study~\cite{Kohlhase:smtl05}, where we semantically preload the course materials for a two-semester course in Computer Science at Jacobs University Bremen and transform them to the {\omdoc} MKM format, so that they can be used in the {\activemath} system~\cite{activemathAIEDJ01}. Another study of converting {\LaTeX} materials for the {\connexions} project is under way.\ednote{say some more} \subsection{Licensing, Download and Setup}\label{sec:setup} \ednote{talk about licensing} The {\stex} packages and classes can be obtained as a self-documenting {\LaTeX} packages: To obtain a package {\meta{package}} download the files \meta{package}|.dtx| and \meta{package}|.ins| from \begin{center} {\url{https://svn.kwarc.info/repos/kwarc/projects/stex/sty/}\meta{package}/} \end{center} To extract the {\LaTeX} package \meta{package}|.sty| and the {\latexml} bindings in \meta{package}|.ltxml|, run the {\LaTeX} formatter on |cmathml.ins|, e.g. by typing |latex cmathml.ins| to a shell. To extract the documentation (the version of this document that goes with the extracted package) run the {\LaTeX} formatter on |cmathml.dtx| e.g. by typing |latex |\meta{package}|.dtx| to a shell. Usually, the {\stex} distribution will also have the newest versions of the files \meta{package}|.sty|, \meta{package}|.ltxml|, and the documentation \meta{package}|.pdf| pre-generated for convenience, so they can be downloaded directly from the URL above. To install the package, copy the file \meta{package}|.sty| somewhere, where {\TeX}/{\LaTeX} can find it and rebuild {\TeX}'s file name database. This is done by running the command |texhash| or |mktexlsr| (they are the same). In |MikTEX|, there is a menu option to do this. \section{Utilities}\label{sec:utilities} To simplify dealing with {\stex} documents, we are providing a small collection of command line utilities, which we will describe here. For details and downloads go to {\url{http://kwarc.info/projects/stex}}. \begin{description} \item[{\tt{msplit}}] splits an {\stex} file into smaller ones (one module per file) \item[{\tt{rf}}] computes the ``reuse factor'', i.e. how often {\stex} modules are reused over a collection of documents \item[{\tt{sgraph}}] visualizes the module graph \item[{\tt{sms}}] computes the {\stex} module signatures for a give {\stex} file \item[{\tt{bms}}] proposes a sensible module structure for an un-annotated {\stex} file \end{description} \DocInclude{cnx} \DocInclude{cmathml} \DocInclude{modules} \DocInclude{statements} \DocInclude{sproof} \DocInclude{omdoc} \DocInclude{presentation} % Stop here if ltxdoc.cfg says \AtEndOfClass{\OnlyDescription} \StopEventually{\newpage \bibliographystyle{alpha} \bibliography{kwarc} \PrintIndex \newpage\PrintChanges \end{document}} \clearpage \pagestyle{headings} % Make TeX shut up. \hbadness=10000 \newcount\hbadness \hfuzz=\maxdimen \bibliographystyle{alpha} \bibliography{../doc/macros/stex,../doc/macros/omdoc} \typeout{% \string # Produce change log with^^J% makeindex -s gglo.ist -o stex.gls stex.glo} \PrintChanges \clearpage % makeindex needs a symbol between the parts of composite page numbers % but we dont want one, so: \typeout{% \string # Produce index with^^J% makeindex -s stex.ist stex.idx} \PrintIndex % Make sure that the index is not printed twice % (ltxdoc.cfg might have a second \PrintIndex command) \let\PrintChanges\relax \let\PrintIndex\relax \end{document} %%% Local Variables: %%% mode: stex %%% TeX-master: t %%% End: % LocalWords: hoc LaTeXML nC CSumLimits cmathml DVI th sproof dtx mikoslides % LocalWords: ltxml pdf texhash mktexlsr MikTEX msplit rf sgraph sms bms un % LocalWords: cnx omdoc