diff options
| author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
|---|---|---|
| committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
| commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
| tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/pkgcheck | |
Initial commit
Diffstat (limited to 'support/pkgcheck')
27 files changed, 5070 insertions, 0 deletions
diff --git a/support/pkgcheck/CHANGES.md b/support/pkgcheck/CHANGES.md new file mode 100644 index 0000000000..912a2e8aaf --- /dev/null +++ b/support/pkgcheck/CHANGES.md @@ -0,0 +1,62 @@ +# General remarks + +- the x.y.z is the version of the Rust source code which follows semantic versioning +- the combination of version and version date designates the version of the package as it is uploaded to CTAN + + +# Changes + +2018-12-09 (1.0.0) + - 1.0.0 First stable version + +2018-12-16 (1.0.0) + - quick_intro.txt: improve wording + - build_ctan_zip.p6: add source files to be included into CTAN zip archive + - devnotes.md: add documentation how to build your own binary +2018-12-26 (1.1.0) + - enhance check for generated files which now could reside in any + subdirectory in the package tree. This changes the format of the + e0019 message slightly where now the path of a generated file will be + displayed in the e0019 message + - change ordering of the sections in the pkgcheck.pdf document. First come the + informational messages, then warnings and error messages and finally fatal messages. + - add a short info to the PDF documention about how to install pkgcheck +2019-01-02 (1.2.0) + - some code simplification + - **New feature**: checks that path names in the TDS zip archive contain the package name + (only exception is a man page path) + - add test cases for messages e0026, e0028 +2019-03-09 (1.3.0) + - recompile with newest http library + - checking URLs + - when checking URLs we try to get the headers first. If this fails we try to get the web page. + - when checking URLs and a redirect has an invalid location containing 127.0.0.1 + then we regard the url as ok + - ignoring .tfm files when checking for duplicates + - now checking for UTF BOMs. If a README file contains a BOM we issue an error message, + if other files contain BOMs we issue a warning message +2019-03-11 (1.4.0) + - recognizing generated files in a .dtx file when they are included using a + filecontents resp. filecontents* environment. +2019-03-14 (1.5.0) + - a README can be a symlink which will now be detected properly + - if a symlink points to a file object outside of the package directory tree error message + e0030 will be issued +2019-03-29 (1.6.0) + - new error message e0031 when a filename contains invalid UTF-8 characters +2019-06-30 (1.7.0) + - when symlinks occur in e0025 they are reported as files + - improved error message e0025 now displaying the paths of the affected + files/directories/symlinks + - if a found http(s) link ends with "`" then the trailing "`" will be discarded + - compiled with rust edition 2018 +2019-07-20 (1.8.0) + - add a check to detect temporary files in the TDS zip archive +2019-07-21 (1.8.1) + - correct typo in e0008 message text +2019-08-07 (1.8.2) + - updating crates, requiring slight code changes in integration.rs + - ignore windows files .nsh and .reg when checking for CRLF line endings + - new error message e0034 for unwanted files in top level directory of a TDS zip archive + - e0020 now reporting only unwanted directory in top level directory of a TDS zip archive + diff --git a/support/pkgcheck/Cargo.toml b/support/pkgcheck/Cargo.toml new file mode 100644 index 0000000000..74b14f05cc --- /dev/null +++ b/support/pkgcheck/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "pkgcheck" +version = "1.8.2" +authors = ["Manfred Lotz <manfred@ctan.org>"] +description = "Command-line tool to check packages uploaded to CTAN." +license = "MIT/Apache-2.0" +edition = "2018" + +[dependencies] +structopt = "0.2" +walkdir = "2.2" +scoped_threadpool = "0.1" +num_cpus = "1" +blake2 = "0.8" +fnv = "1.0.6" +unbytify = "0.2" +regex = "1" +glob = "0.3" +colored = "1" +linkify = "0.4" +reqwest = "0.9" +lazy_static = "1" +tempdir = "0.3" +assert_cmd = "0.11" +predicates = "1.0" +openssl-probe = "0.1.2" +pest = "^2.1" +pest_derive = "2.1" +escargot = "0.5" + +threadpool = "1.7.1" +unicode-bom = "1" + + +url = "2.1" diff --git a/support/pkgcheck/LICENSE-APACHE b/support/pkgcheck/LICENSE-APACHE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/support/pkgcheck/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/support/pkgcheck/LICENSE-MIT b/support/pkgcheck/LICENSE-MIT new file mode 100644 index 0000000000..c0f44fc630 --- /dev/null +++ b/support/pkgcheck/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2019 Manfred Lotz + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/support/pkgcheck/README.md b/support/pkgcheck/README.md new file mode 100644 index 0000000000..96d6b16716 --- /dev/null +++ b/support/pkgcheck/README.md @@ -0,0 +1,66 @@ +--- +pkgcheck utility + +Author: Manfred Lotz, <manfred@ctan.org> + +License: Apache License, Version 2.0 or MIT License + +--- + + +# Overview + +`pkgcheck` is a utility which the author uses to check uploaded packages to CTAN before +installing them. It is a binary running on Linux only. + +There is no Windows version planned. + + +# Dependencies + +The `pkgcheck` binary is a 64-bit statically linked binary, and thus it should run also on +older Linux versions. + +It uses the following external programs: + +- `pdfinfo` for checking pdf documents +- `unzip` for temporarily unpacking a TDS zip archive + +# Installing the binary + +Copy the binary from `bin/pkgcheck` to a suitable location on your hard disk, and +(recommended) make sure the directory is in the `PATH` or call `pkgcheck` using an +absolute path name. + +# Documentation + +The documentation is `docs/pkgcheck.pdf`. It contains +a description of all fatal, error, warning and information messages. + + +# Build the documentation + +Run either `xelatex` or `lualatex`. Note that `-shell-escape` is required. + + +``` +cd docs +lualatex -shell-escape pkgcheck.tex +``` + + + +# License + +Licensed under either of + +- Apache License, Version 2.0 + + - See file LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0 + +- MIT license + + - See file LICENSE-MIT or http://opensource.org/licenses/MIT + +at your option. + diff --git a/support/pkgcheck/bin/pkgcheck b/support/pkgcheck/bin/pkgcheck Binary files differnew file mode 100755 index 0000000000..aedf999a10 --- /dev/null +++ b/support/pkgcheck/bin/pkgcheck diff --git a/support/pkgcheck/docs/errorsd.tex b/support/pkgcheck/docs/errorsd.tex new file mode 100644 index 0000000000..1557460ea1 --- /dev/null +++ b/support/pkgcheck/docs/errorsd.tex @@ -0,0 +1,332 @@ +\hypertarget{e0001----bad-characters-in-file-name}{% +\subsection{E0001 -\/- Bad characters in file +name}\label{e0001----bad-characters-in-file-name}} + +File name should not contain non-ascii characters. Additionally, file +names should not contain control characters or other characters which +may have a special meaning for UNIX shells. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nounixspecialcharacters}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nounixspecialcharacters} + +\hypertarget{e0002----file-permissions}{% +\subsection{E0002 -\/- File +Permissions}\label{e0002----file-permissions}} + +Files submitted to CTAN should be world readable. + +Only files that are truly executable (like scripts and binaries) should +be marked as such. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions} + +\hypertarget{e0003----readme-is-not-a-text-file}{% +\subsection{E0003 -\/- README is not a text +file}\label{e0003----readme-is-not-a-text-file}} + +The \texttt{README} file specified in the error message must be a text +file but it isn't. + +\hypertarget{e0004----empty-directory-not-allowed}{% +\subsection{E0004 -\/- Empty directory not +allowed}\label{e0004----empty-directory-not-allowed}} + +Empty directories are considered as rubbish, and are usually not +accepted as part of a package. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles} + +\hypertarget{e0005----empty-files-not-allowed}{% +\subsection{E0005 -\/- Empty files not +allowed}\label{e0005----empty-files-not-allowed}} + +Empty files are considered as rubbish, and are usually not accepted as +part of a package. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noemptyfiles} + +\hypertarget{e0006----hidden-directories-not-allowed}{% +\subsection{E0006 -\/- Hidden directories not +allowed}\label{e0006----hidden-directories-not-allowed}} + +A package should not contain hidden directories. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} + +\hypertarget{e0007----hidden-files-not-allowed}{% +\subsection{E0007 -\/- Hidden files not +allowed}\label{e0007----hidden-files-not-allowed}} + +A package should not contain hidden files. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} + +\hypertarget{e0008----temporary-file-detected}{% +\subsection{E0008 -\/- Temporary file +detected}\label{e0008----temporary-file-detected}} + +A temporary file was detected. These are typically files created by TeX +\& friends and should not be part of a package. + +Temporary files will also be detected in a TDS zip archive. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} + +\hypertarget{e0009----package-doesnt-contain-a-readme-file}{% +\subsection{E0009 -\/- Package doesn't contain a README +file}\label{e0009----package-doesnt-contain-a-readme-file}} + +A package must contain at least one of \texttt{README}, +\texttt{README.md} or \texttt{README.txt} file. + +For more details refer to: +\href{http://mirrors.ibiblio.org/CTAN/help/ctan/CTAN-upload-addendum.html\#readme}{http://mirrors.ibiblio.org/CTAN/help/ctan/CTAN-upload-addendum.html\#readme} + +\hypertarget{e0010----broken-symlink-detected}{% +\subsection{E0010 -\/- Broken symlink +detected}\label{e0010----broken-symlink-detected}} + +A broken symlink was detected. + +\hypertarget{e0011----wrong-permission-for-directory}{% +\subsection{E0011 -\/- Wrong permission for +directory}\label{e0011----wrong-permission-for-directory}} + +Directories should have rwx for the owner and at least \texttt{r-x} for +others (i.e. world readable). + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions} + +\hypertarget{e0012----crlf-line-endings-detected}{% +\subsection{E0012 -\/- CRLF line endings +detected}\label{e0012----crlf-line-endings-detected}} + +The file specified in the error message contains CRLF line endings. Text +files should have UNIX style line endings. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf} + +\hypertarget{e0013----socket-special-fie-detected}{% +\subsection{E0013 -\/- Socket special fie +detected}\label{e0013----socket-special-fie-detected}} + +The file specified in the error message is a socket special file which +is not allowed. + +\hypertarget{e0014----fifo-special-file-detected}{% +\subsection{E0014 -\/- Fifo special file +detected}\label{e0014----fifo-special-file-detected}} + +The file specified in the error message is a fifo special file which is +not allowed. + +\hypertarget{e0015----bloch-device-file-detected}{% +\subsection{E0015 -\/- Bloch device file +detected}\label{e0015----bloch-device-file-detected}} + +The file specified in the error message is a block device file which is +not allowed. + +\hypertarget{e0016----character-device-file-detected}{% +\subsection{E0016 -\/- Character device file +detected}\label{e0016----character-device-file-detected}} + +The file specified in the error message is a character device file which +is not allowed. + +\hypertarget{e0017----pdf-document-is-in-error}{% +\subsection{E0017 -\/- PDF document is in +error}\label{e0017----pdf-document-is-in-error}} + +The PDF document mentioned in the message is in error. + +\texttt{pdfinfo} will be run to check if a PDF document can be read. +Message \texttt{E0017} will be followed by the error messages from +\texttt{pdfinfo}. + +Example: + +\begin{verbatim} +I0002 Checking package files in directory somepkg +E0017 PDF error detected in somepkg/sompkg.pdf +Syntax Error (1293042): Illegal character ')' +Syntax Error: Couldn't find trailer dictionary +Syntax Error (1293042): Illegal character ')' +Syntax Error: Couldn't find trailer dictionary +Syntax Error: Couldn't read xref table +\end{verbatim} + +\hypertarget{e0018----unwanted-directory-detected}{% +\subsection{E0018 -\/- Unwanted directory +detected}\label{e0018----unwanted-directory-detected}} + +A directory was detected which should not be part of a package. Example: +\texttt{\_\_MACOSX} + +\hypertarget{e0019----generated-file-detected}{% +\subsection{E0019 -\/- Generated file +detected}\label{e0019----generated-file-detected}} + +In order to avoid redundancy we don't want to have included files in a +package which easily can be generated from other files in the +submission. + +Exceptions are the \texttt{README} files of the package, i.e. +\texttt{README}, \texttt{README.md} or \texttt{README.txt}. + +Starting with version 1.1.0 \texttt{pkgcheck} also detects generated +files if they are in a different directory in the package. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#nogeneratedfiles} + +\hypertarget{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}{% +\subsection{E0020 -\/- Unwanted directory detected in the top level +directory in TDS zip +archive}\label{e0020----unwanted-directory-detected-in-the-top-level-directory-in-tds-zip-archive}} + +A top level directory of a TDS archive should only contain all or some +of the following directories: + +\begin{itemize} +\tightlist +\item + \texttt{tex} +\item + \texttt{fonts} +\item + \texttt{metafont} +\item + \texttt{metapost} +\item + \texttt{bibtex} +\item + \texttt{scripts} +\item + \texttt{doc} +\item + \texttt{source} +\end{itemize} + +\hypertarget{e0021----error-when-reading-a-file}{% +\subsection{E0021 -\/- Error when reading a +file}\label{e0021----error-when-reading-a-file}} + +An error was encountered when reading the file specified in the message. + +\hypertarget{e0022----check-of-an-url-in-a-readme-file-failed}{% +\subsection{E0022 -\/- Check of an URL in a README file +failed}\label{e0022----check-of-an-url-in-a-readme-file-failed}} + +URL checking is in effect. An error occcured when trying to retrieve an +URL which was found in the specified \texttt{README} file. + +\hypertarget{e0023----follow-up-error-when-trying-to-read-a-directory-with-insufficient-permissions}{% +\subsection{E0023 -\/- Follow up error when trying to read a directory +with insufficient +permissions}\label{e0023----follow-up-error-when-trying-to-read-a-directory-with-insufficient-permissions}} + +Error which is a follow-up error. For instance, when a directory could +not be read. + +\hypertarget{e0024----tds-zip-archive-has-wrong-permissions}{% +\subsection{E0024 -\/- TDS zip archive has wrong +permissions}\label{e0024----tds-zip-archive-has-wrong-permissions}} + +The TDS zip archive should have at least \texttt{r-\/-} for the owner +and at least \texttt{r-\/-} for others (i.e. world readable). + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions} + +\hypertarget{e0025----duplicate-names-when-ignoring-letter-case-for-files-or-directories}{% +\subsection{E0025 -\/- Duplicate names when ignoring letter case for +files or +directories}\label{e0025----duplicate-names-when-ignoring-letter-case-for-files-or-directories}} + +As there are operating systems which do not distinguish between +\texttt{myfile} and \texttt{MYFILE} we don't want to have file names in +a directory which are the same after converting to lower case. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filenames}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filenames} + +\hypertarget{e0026----files-not-in-tds-or-different-in-tds-and-non-install-tree}{% +\subsection{E0026 -\/- Files not in TDS or different in TDS and +non-install +tree}\label{e0026----files-not-in-tds-or-different-in-tds-and-non-install-tree}} + +The file mentioned in the error message is either not existing in the +TDS zip archive, or it is different to the one in the non-install tree + +\hypertarget{e0027----an-io-error-occured}{% +\subsection{E0027 -\/- An I/O error +occured}\label{e0027----an-io-error-occured}} + +Some kind of I/O error occured. If you believe there is an error in +\texttt{pkgcheck} please contact the author. + +\hypertarget{e0028----a-path-name-in-a-tds-zip-archive-must-contain-the-package-name}{% +\subsection{E0028 -\/- A path name in a TDS zip archive must contain the +package +name}\label{e0028----a-path-name-in-a-tds-zip-archive-must-contain-the-package-name}} + +The path names in a TDS zip archive must contain the package name. + +\textbf{Example:} Assume a package \texttt{somepkg}. Then path names +should look like follows: + +\begin{verbatim} +tex/latex/somepkg/somepkg.cls +doc/latex/somepkg/README +source/latex/somepkg/somepkg.dtx +... +\end{verbatim} + +\hypertarget{e0029----readme-file--encoding-with-bom-detected}{% +\subsection{\texorpdfstring{E0029 -\/- README file: encoding with BOM +detected}{E0029 -\/- README file: encoding with BOM detected}}\label{e0029----readme-file--encoding-with-bom-detected}} + +A README file should be either ASCII or UTF-8 without BOM(byte order +mark) + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#readme}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#readme} + +\hypertarget{e0030----a-symlink-was-found-which-points-outside-of-the-package-directory-tree}{% +\subsection{E0030 -\/- A symlink was found which points outside of the +package directory +tree}\label{e0030----a-symlink-was-found-which-points-outside-of-the-package-directory-tree}} + +A symlink must not point to a file or directory outside of the package +directory tree. + +\hypertarget{e0031----file-name-contains-invalid-utf-8-characters}{% +\subsection{E0031 -\/- File name contains invalid UTF-8 +character(s)}\label{e0031----file-name-contains-invalid-utf-8-characters}} + +A file name contains invald UTF-8 character(s). + +\hypertarget{e0033----error-when-unpacking-tds-archive}{% +\subsection{E0033 -\/- Error when unpacking tds +archive}\label{e0033----error-when-unpacking-tds-archive}} + +In order to investigate the contents of the TDS zip archive +\texttt{pkgcheck} unpacks the TDS zip archive to a temporary location +which failed for the reason given in the error message. + +\hypertarget{e0034----unwanted-file-detected-in-the-top-level-directory-in-tds-zip-archive}{% +\subsection{E0034 -\/- Unwanted file detected in the top level directory +in TDS zip +archive}\label{e0034----unwanted-file-detected-in-the-top-level-directory-in-tds-zip-archive}} + +A top level directory of a TDS archive should only contain certain +directories but no files. diff --git a/support/pkgcheck/docs/fatald.tex b/support/pkgcheck/docs/fatald.tex new file mode 100644 index 0000000000..b4c175a7c0 --- /dev/null +++ b/support/pkgcheck/docs/fatald.tex @@ -0,0 +1,39 @@ +\hypertarget{f0001----specify-a-directory-to-check-use-option--d}{% +\subsection{F0001 -\/- Specify a directory to check (use option +-d)}\label{f0001----specify-a-directory-to-check-use-option--d}} + +\texttt{pkgcheck} was called without any options. Use option \texttt{-d} +to check a directory + +\hypertarget{f0002----specified-directory-does-not-exist-exiting}{% +\subsection{F0002 -\/- Specified directory does not exist. +Exiting...}\label{f0002----specified-directory-does-not-exist-exiting}} + +The directory specified at the command line does not exit. + +\hypertarget{f0003----specified-tds-archive-does-not-exist-or-is-no-file}{% +\subsection{F0003 -\/- Specified TDS archive does not exist or is no +file}\label{f0003----specified-tds-archive-does-not-exist-or-is-no-file}} + +Specify a valid TDS zip archive when calling \texttt{pkgcheck} + +\hypertarget{f0004----the-file-specified-as-tds-archive-is-no-zip-archive}{% +\subsection{F0004 -\/- The file specified as TDS archive is no zip +archive}\label{f0004----the-file-specified-as-tds-archive-is-no-zip-archive}} + +Specify a valid TDS zip archive when calling \texttt{pkgcheck} + +\hypertarget{f0005----bad-file-name-for-the-zip-archive}{% +\subsection{F0005 -\/- Bad file name for the zip +archive}\label{f0005----bad-file-name-for-the-zip-archive}} + +pkgcheck detected that the file name of the TDS zip archive doesn't end +with \texttt{.tds.zip} + +\hypertarget{f0006----unknown-error-code-specified-with-option--e-resp----explain-exiting}{% +\subsection{F0006 -\/- Unknown error code specified with option -e resp. +-\/-\/-explain. +Exiting...}\label{f0006----unknown-error-code-specified-with-option--e-resp----explain-exiting}} + +\texttt{pkgcheck} was called with option \texttt{-e} resp. +\texttt{-\/-explain}, and an unknown error code was specified. diff --git a/support/pkgcheck/docs/informationd.tex b/support/pkgcheck/docs/informationd.tex new file mode 100644 index 0000000000..88a2305f48 --- /dev/null +++ b/support/pkgcheck/docs/informationd.tex @@ -0,0 +1,51 @@ +\hypertarget{i0001----successfully-converted-from-crlf-to-lf}{% +\subsection{I0001 -\/- Successfully converted from CRLF to +LF}\label{i0001----successfully-converted-from-crlf-to-lf}} + +Just an information that \texttt{pkgcheck} has successfully converted a +file from CRLF to LF line endings + +\hypertarget{i0002----checking-package-files-in-directory}{% +\subsection{I0002 -\/- Checking package files in +directory}\label{i0002----checking-package-files-in-directory}} + +Just an information that \texttt{pkgcheck} starts checking the package +files in the unzipped directory trees + +\hypertarget{i0003----checking-tds-zip-archive}{% +\subsection{I0003 -\/- Checking TDS zip +archive}\label{i0003----checking-tds-zip-archive}} + +Just an information that \texttt{pkgcheck} starts checking the TDS zip +archive + +\hypertarget{i0004----correcting-line-endings-for-file}{% +\subsection{I0004 -\/- Correcting line endings for +file}\label{i0004----correcting-line-endings-for-file}} + +The file had CRLF line ending and will be corrected to have LF (Unix +like) line endings. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#crlf} + +\hypertarget{i0005----corrections-permissions-for-file-or-directory}{% +\subsection{I0005 -\/- Corrections permissions for file or +directory}\label{i0005----corrections-permissions-for-file-or-directory}} + +\texttt{pkgcheck} corrects wrong permsissions for package files and +directories. It runs the \texttt{chmod} command in verbose mode. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#filepermissions} + +\hypertarget{i0006----files-having-one-of-the-following-file-name-endings-are-regarded-as-temporary}{% +\subsection{I0006 -\/- Files having one of the following file name +endings are regarded as +temporary}\label{i0006----files-having-one-of-the-following-file-name-endings-are-regarded-as-temporary}} + +Option -\/-show-temp-endings was used, and pkgcheck prints a list of +temporary file endings and their meanings. + +For more details refer to: +\href{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles}{http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html\#noauxfiles} diff --git a/support/pkgcheck/docs/pkgcheck.pdf b/support/pkgcheck/docs/pkgcheck.pdf Binary files differnew file mode 100644 index 0000000000..23e52fafe3 --- /dev/null +++ b/support/pkgcheck/docs/pkgcheck.pdf diff --git a/support/pkgcheck/docs/pkgcheck.tex b/support/pkgcheck/docs/pkgcheck.tex new file mode 100644 index 0000000000..ab96cc390a --- /dev/null +++ b/support/pkgcheck/docs/pkgcheck.tex @@ -0,0 +1,375 @@ +\listfiles +\documentclass[fontsize=10pt,paper=a4,abstract=on]{scrartcl} + +\DeclareTOCStyleEntry[dynnumwidth=true]{default}{subsection} + +\usepackage{unicode-math} + +\usepackage[RM={StylisticSet=3}]{plex-otf} + +\usepackage[english]{babel} +\usepackage[autostyle]{csquotes} + +\usepackage{array,multido} +\usepackage{booktabs} % for examples +%\usepackage{xltabular} % for examples +\usepackage{listings} +\lstset{columns=fixed,basicstyle=\ttfamily\small} +\usepackage[table]{xcolor} +\usepackage{xspace} + +\usepackage{longtable} + +\usepackage{minted} +\usemintedstyle{emacs} + +\pagestyle{headings} + +\usepackage[colorlinks, pdfusetitle, hyperfootnotes=false]{hyperref} +% define \code for url-like breaking of typewriter fragments. +\ifx\nolinkurl\undefined \let\code\url \else \let\code\nolinkurl \fi + +% Define \cs to prepend a backslash, and be unbreakable: +\DeclareRobustCommand\cs[1]{\mbox{\texttt{\char`\\#1}}} + + +\newcommand{\pkgcheck}{\texttt{pkgcheck}\xspace} +\newcommand{\msg}[1]{\texttt{#1}\xspace} + +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} + +% \pdfinfo{ +% /Author (Manfred Lotz) +% /Title pkgcheck Utility +% } + +%\title{pkgcheck utility} +\include{title} +\author{Manfred Lotz} + + +\begin{document} +\maketitle +\tableofcontents + +\begin{abstract} + This document describes the \pkgcheck command line utility which is used by the + author when checking uploaded packages to CTAN. +\end{abstract} + + +\section{Introduction} + +Uploaded packages to CTAN must satisfy various requirements in order to get +installed on CTAN. + +A first introduction is given here \url{https://ctan.org/help/upload-pkg}. + +Even more details are to be found in the excellent CTAN-upload addendum +\url{https://ctan.org/file/help/ctan/CTAN-upload-addendum} written by Petra Rübe-Pugliese. + +The \pkgcheck utility which runs on Linux systems only checks those requirements +which can be checked by a program. + +\section{pkgcheck utility} + +The \pkgcheck utility is a compiled program written in the Rust programming +language. It runs in a Linux environment. Currently, Windows is not supported. +Simply, because the author doesn't use Windows at all. + +It will be invoked from the command line, and any error or +warning message has a certain message id. \pkgcheck offers an option to get more +information for a certain error. + +\section{Requirements} + +\pkgcheck doesn't have any special runtime requirements. + +The \pkgcheck is a 64-bit statically linked binary, and should work an any 64-bit Linux. +It is available in the repository in directory \verb|bin/|. + +Currently, the only external programs required are: +\begin{itemize} +\item \verb|/usr/bin/unzip| + +Used only, when a TDS zip archive will be extracted. + +\item \verb|/usr/bin/pdfinfo| + +Used only, when a PDF document will be checked. + +\end{itemize} + +\section{Installation} + +Copy the binary from \verb|bin/pkgcheck| to a suitable location on your hard disk, and +(recommended) make sure the directory is in the \verb|PATH| or call \verb|pkgcheck| using an +absolute path name. + +\section{Utility usage} + + +\subsection{Help option} + +Running \verb|pkgcheck --help| shows the available command line options. + +Here a sample output: + +\begin{verbatim} +pkgcheck 1.0.0 +Manfred Lotz <manfred@ctan.org> +A checker for uploaded packages to CTAN. + +USAGE: + pkgcheck [FLAGS] [OPTIONS] + +FLAGS: + -L, --correct-crlf Correct CRLF line endings + -C, --correct-perms Correct permissions + --explain-all Explains all error or warning messages + -h, --help Prints help information + -I, --ignore-dupes Ignore dupes + --no-colors Don't display messages in color + --show-temp-endings Show file endings for temporary files + --urlcheck Check URLs found in README files + -V, --version Prints version information + -v, --verbose Verbose operation? + +OPTIONS: + -e, --explain <explain> Explain error or warning message + -d, --package-dir <pkg_dir> Package directory + -T, --tds-zip <tds_zip> tds zip archive + +\end{verbatim} + + +\subsection{Check a package} + +A package for CTAN is supposed to be uploaded as a ZIP or a g-zipped +tar archive. The package must have a top level directory. + +After unpacking the archive of a package \texttt{mypkg} into directory +\texttt{mypkg/} it can be checked by running \pkgcheck with +option verb|--package-dir| or shorter \verb|-d|. + + +\begin{minted}[linenos=true,frame=single,fontsize=\footnotesize]{sh} + pkgcheck -d mypkg +\end{minted} + +\pkgcheck returns 1 if there are any errors, otherwise 0. + + + +\subsection{Check a package which has a TDS archive} + +If a package contains a TDS ZIP archive it is supposed to be in the +top level directory of a package. + +In order to check the TDS ZIP archive the option +\verb|-T <tds_zip>| or \verb|--tds-zip <tds_zip>| can be used. + +Please note that a TDS ZIP archive will always be checked together with +the non-install tree of the package which means that +\verb|--tds-zip| requires option \verb|--package-dir| as well. + +Checking package \texttt{mypkg} \pkgcheck will be invoked like follows: + +\begin{minted}[linenos=true,frame=single,fontsize=\footnotesize]{sh} + pkgcheck -d mypkg -T mypkg.tds.zip +\end{minted} + + +As before \pkgcheck returns 1 if there are any errors, otherwise 0. + + +\subsection{Pkgcheck messages} + +\pkgcheck issues three kind of messages + +\begin{itemize} +\item Information messages +\item Warning messages +\item Error messages +\end{itemize} + +Messages have unique ids and the detailed explanation of a message can be either +looked up in this document, or it can be displayed by using command line option +\verb|--explain| or \verb|-e|. + + +\begin{minted}[linenos=true,label=Example,frame=single,fontsize=\footnotesize]{sh} + pkgcheck --explain e0012 +\end{minted} + + + +\subsection{Duplicate files} + +By default, \pkgcheck detects duplicate files in a package. This could be +disabled by using command line switch \verb|--ignore-dups| or shorter \verb|-I|. + +\subsection{Permissions} + +\pkgcheck offers the option \verb|--correct-crlf| or shorter \verb|-L| to +correct wrong permissions in a package. + + +\subsection{CRLF line endings} + +\pkgcheck detects CRLF line endings in text files as good as it can. +It reads up to 1 MB to check for CRLF line endings. + +Option \verb|--correct-crlf| or for short \verb|-L| can be used to convert a +file from CRLF to LF line endings. + +\subsection{Help options} + +\begin{itemize} +\item \verb|-V| + + Outputs \pkgcheck's version number. + +\item \verb|--help| + + \verb|--help| shows the available command line options. + +\begin{verbatim} +pkgcheck 1.0.0 +Manfred Lotz <manfred@ctan.org> +A checker for uploaded packages to CTAN. + +USAGE: + pkgcheck [FLAGS] [OPTIONS] + +FLAGS: + -L, --correct-crlf Correct CRLF line endings + -C, --correct-perms Correct permissions + --explain-all Explains all error or warning messages + -h, --help Prints help information + -I, --ignore-dupes Ignore dupes + --no-colors Don't display messages in color + --show-temp-endings Show file endings for temporary files + --urlcheck Check URLs found in README files + -V, --version Prints version information + -v, --verbose Verbose operation? + +OPTIONS: + -e, --explain <explain> Explain error or warning message + -d, --package-dir <pkg_dir> Package directory + -T, --tds-zip <tds_zip> tds zip archive + +\end{verbatim} + +\item \verb|--explain <explain| + + This option explains an error message in more detail. + + Example: + +\begin{minted}[linenos=true,frame=single,fontsize=\footnotesize]{sh} + pkgcheck -e e0012 +\end{minted} + + +\item \verb|--explain-all| + +Outputs a list of explanations of all messages. + +\item \verb|--show-temp-endings| + + Outputs a list of all file name endings which \pkgcheck uses to detect + temporary files. +\end{itemize} + +%\subsection{Other options} + +\section{About checking file types} + +\pkgcheck determines, similar to the UNIX \verb|file| command, the type of file. +This is required before for example checking permissions or complaining that a +text file has CRLF line endings. + +It is very important to note that determining file types is not bullet proof. +So, it might happen in some cases that \pkgcheck makes mistakes when determining +a file type. This could lead to a subseqent mistake when complaining about an +x-bit, or complaining about CRF line ending. + + + + + + +\section{About permissions checking} + +From an installation point of view the files and directories of a package +\begin{itemize} +\item must be at least world readable +\item must be writable by owner or group +\item must not have the x-bit on for the owner if the file isn't an executable, + i.e. a script or binary +\end{itemize} + +The reason for this minimal requirement is that the installation utility used by +the CTAN team (which by the way was written by Rainer Schöpf a long time ago) +sets permissions correctly if the owner permission is set correctly. + +Examples: + +\begin{itemize} +\item \verb|README.md| with \texttt{666} is ok because the installation utility converts + the permission to 664 +\item \verb|README.md| with \texttt{660} is wrong because the installation utility + wouldn't have access to the file +\item \verb|some.pdf| with \texttt{744} would be wrong because a PDF document must not + have the x-bit on for the owner +\end{itemize} + +Because of the smartness of the installation utility \pkgcheck does check +minimal requirements only, i.e some weird looking permissions like the \texttt{666} above +are accepted. + +\section{Different kind of messages} + +\begin{description} +\item[Innnn] Informational messages + + These are message which announce \pkgcheck actions. + +\item[Fnnnn] Fatal messages + + Fatal messages report unrecoverable errors. In this case \pkgcheck's only + option is to terminate. If for example, the package directory specified at + the command line doesn't exist then the only option is to terminate. + +\item[Ennnn] Error messages + + Error messages report errors which must be fixed before installing a package. + +\item[Wnnnn] Warning messages + + Warning messages denote possible errors depending upon the situation. + + For example, for a font package having many duplicate files might be ok. For + another package it could be regarded as an error. + + + + +\end{description} +\section{Informational messages} +\input{informationd} + +\section{Warning messages} +\input{warningsd} + +\section{Error messages} +\input{errorsd} + +\section{Fatal messages} +\input{fatald} + +\end{document} + + diff --git a/support/pkgcheck/docs/title.tex b/support/pkgcheck/docs/title.tex new file mode 100644 index 0000000000..f13bf6193c --- /dev/null +++ b/support/pkgcheck/docs/title.tex @@ -0,0 +1 @@ +\title{pkgcheck Utility, v1.8.2} diff --git a/support/pkgcheck/docs/warningsd.tex b/support/pkgcheck/docs/warningsd.tex new file mode 100644 index 0000000000..7b8c676e23 --- /dev/null +++ b/support/pkgcheck/docs/warningsd.tex @@ -0,0 +1,39 @@ +\hypertarget{w0001----archive-as-package-file-detected}{% +\subsection{W0001 -\/- Archive as package file +detected}\label{w0001----archive-as-package-file-detected}} + +Usually a CTAN package should not contain archives. An exception are +situations where, for example, the source code of a package is kept in a +separate zip archive. + +\hypertarget{w0002----duplicate-files-detected}{% +\subsection{W0002 -\/- Duplicate files +detected}\label{w0002----duplicate-files-detected}} + +Duplicate files were detected which are listed right after this message. + +The message is a warning message as something like this could not be +seen as an error in general. + +\hypertarget{w0003----same-named-files-detected-in-the-package-tree}{% +\subsection{W0003 -\/- Same named files detected in the package +tree}\label{w0003----same-named-files-detected-in-the-package-tree}} + +We like to have unique file names over the whole package directory tree. +When we discover same named files we report it as a warning. Common +names like \texttt{README}, \texttt{README.txt}, \texttt{README.md}, +\texttt{Makefile}, \texttt{Makefile.in}, \texttt{Makefile.am} and +\texttt{makefile} are ignored when checking. + +For more details refer to: +\href{http://mirror.utexas.edu/ctan/help/ctan/CTAN-upload-addendum.html\#uniquefilenames}{http://mirror.utexas.edu/ctan/help/ctan/CTAN-upload-addendum.html\#uniquefilenames} + +\hypertarget{w0004-----encoding-with-bom-detected}{% +\subsection{\texorpdfstring{W0004 -\/- encoding with BOM +detected}{W0004 -\/- encoding with BOM detected}}\label{w0004-----encoding-with-bom-detected}} + +A UTF encoded package file contains a BOM (byte order mark). Currently, +we issues a warning. + +Nevertheless, the CTAN team discourages uses of BOM. Please be aware, +that in some future time this could be reagarded as an error. diff --git a/support/pkgcheck/quick_intro.txt b/support/pkgcheck/quick_intro.txt new file mode 100644 index 0000000000..ae2d1a4dea --- /dev/null +++ b/support/pkgcheck/quick_intro.txt @@ -0,0 +1,24 @@ + +You want to check your package before uploading to CTAN? + + +1. Build the zip archive + +2. Make sure your umask is set to 022 + +3. Assuming your package's name is mypkg and your zip archive is mypkg.zip + + Unpack mypkg.zip (which should contain a top level directory mypkg/), + so that you obtain again a directory mypkg/ containing the files of your package. + + unzip mypkg.zip + + Now run `pkgcheck` + + `pkgcheck -d mypkg` + + +If you want to check URLs in README file run + + `pkgcheck --urlcheck -d mypkg` + diff --git a/support/pkgcheck/src/filemagic.rs b/support/pkgcheck/src/filemagic.rs new file mode 100644 index 0000000000..40edd18351 --- /dev/null +++ b/support/pkgcheck/src/filemagic.rs @@ -0,0 +1,422 @@ +#![allow(clippy::trivial_regex)] + +use regex::bytes::Regex; +use regex::bytes::RegexSet; + +use std::fs; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::path::Path; + +use std::os::unix::fs::FileTypeExt; + +use unicode_bom::Bom; + +// +// File signatures links +// - https://asecuritysite.com/forensics/magic +// - https://filesignatures.net/ +// - https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json + +#[derive(Debug, PartialEq)] +pub enum Mimetype { + Binary, + Script, + ScriptCRLF, + Pdf, + Archive, + Zip, + Text, + TextCRLF, + Data, + Unknown, + BlockDevice, + CharDevice, + Directory, + Symlink, + Fifo, + Socket, + Zerofile, + VeryShort, + BOM(Bom), +} + +pub struct Filetype { + buffer: Vec<u8>, +} + +fn is_binary_data(vec: &[u8], len: usize) -> bool { + for v in vec.iter().take(len) { + if *v <= 8 { + return true; + } + } + + false +} + +fn _is_crlf(buffer: &[u8], len: usize) -> bool { + let mut cr = 0; + let mut lf = 0; + + const CR: u8 = 0x0d; // 13 + const LF: u8 = 0x0a; // 10 + + for c in buffer.iter().take(len) { + if *c == LF { + lf += 1; + } else if *c == CR { + cr += 1; + } + } + + let diff: i32 = cr - lf; + if cr > 0 && diff == 0 { + return true; + } + + //println!("cr: {}, lf: {}", cr, lf); + // Heuristics: we accept if only a few lines are not CRLF + match (cr, lf) { + (0, _lf) => return false, + (_cr, 0) => return true, + (cr, _lf) => { + if cr > 500 && diff.abs() < 3 { + return true; + } + } + } + + false +} + +fn is_crlf(buffer: &[u8], len: usize) -> bool { + let mut seen_cr = false; + let mut n_crlf = 0; + let mut n_lf = 0; + let mut n_cr = 0; + + const CR: u8 = 0x0d; // CR 0x0D 13 \r + const LF: u8 = 0x0a; // LF 0x0A 10 \n + + for c in buffer.iter().take(len) { + if *c == LF { + if seen_cr { + n_crlf += 1; + } else { + n_lf += 1; + } + } else if seen_cr { + n_cr += 1; + } + + seen_cr = *c == CR; + } + + // println!("cr: {}, lf: {}, crlf: {}", n_cr, n_lf, n_crlf); + // if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0) + // --> no line terminators + if n_crlf == 0 && n_cr == 0 && n_lf == 0 { + return false; + } + + if n_crlf > 0 || n_cr > 0 { + return true; + } + + false +} + +impl Filetype { + pub fn new() -> Self { + Filetype { + buffer: vec![0; 1024 * 1024], + } + } + + pub fn analyze(&mut self, fname: &str) -> Result<Mimetype, io::Error> { + // Result<Err,Mimetype> { + let path = Path::new(fname); + + if let Some(ft) = get_filetype(path) { + return Ok(ft); + } + + let metadata = fs::symlink_metadata(fname)?; + let file_length: usize = metadata.len() as usize; + + if file_length == 0 { + return Ok(Mimetype::Zerofile); + } + + if metadata.len() == 1 { + return Ok(Mimetype::VeryShort); + } + + let mut hdl_in = File::open(path)?; + + let mut bytes_read: usize = hdl_in.read(&mut self.buffer[0..262])?; + + // PostScript signatures + // - %!PS-Adobe-1.0, %!PS-Adobe-2.0, %!PS-Adobe-3.0, %!PS-Adobe-3.1 + // - %! and a line feed + if bytes_read >= 4 && &self.buffer[0..4] == b"%!PS" { + return Ok(Mimetype::Data); + } + + // - %!\r\n%%BoundingBox: + let re: Regex = Regex::new(r"^(?-u)%!(\x0d\x0a|\x0A)%%BoundingBox").unwrap(); + if bytes_read >= 20 && re.is_match(&self.buffer) { + return Ok(Mimetype::Data); + } + + if bytes_read >= 4 && &self.buffer[0..4] == b"%PDF" { + return Ok(Mimetype::Pdf); + } + + if bytes_read >= 5 && &self.buffer[0..5] == b"<?php" { + return Ok(Mimetype::Script); + } + + // rtf document + if bytes_read >= 6 && &self.buffer[0..6] == b"\x7B\x5C\x72\x74\x66\x31" { + return Ok(Mimetype::Data); + } + + // ZOO archive http://fileformats.archiveteam.org/wiki/ZOO + if bytes_read >= 60 && &self.buffer[20..24] == b"\xDC\xA7\xC4\xFD" { + return Ok(Mimetype::Archive); + } + + let bom: Bom = Bom::from(&self.buffer[0..]); + + if bom != Bom::Null { + return Ok(Mimetype::BOM(bom)); + } + + if is_binary_data(&self.buffer, bytes_read) { + match analyze_binary(&self.buffer) { + Some(Mimetype::Zip) => { + if fname.ends_with(".cdy") { + return Ok(Mimetype::Data); + } else { + return Ok(Mimetype::Zip); + } + } + Some(mt) => return Ok(mt), + None => return Ok(Mimetype::Unknown), + } + } + + // https://en.wikipedia.org/wiki/BinHex + if bytes_read >= 200 + && self + .buffer + .starts_with(b"(This file must be converted with BinHex 4.0)") + { + return Ok(Mimetype::Binary); + } + + if bytes_read < file_length { + if let Ok(rb) = hdl_in.read(&mut self.buffer[262..]) { + bytes_read += rb + } + } + + let crlf = is_crlf(&self.buffer, bytes_read); + + if bytes_read >= 5 && (self.buffer.starts_with(b"#!") || self.buffer.starts_with(b"<?php")) + { + if crlf { + return Ok(Mimetype::ScriptCRLF); + } else { + return Ok(Mimetype::Script); + } + } + + let is_script = Regex::new(r"\.(csh|ksh|py|pl|sh|tcsh)$") + .unwrap() + .is_match(fname.as_bytes()); + + match (crlf, is_script) { + (false, false) => Ok(Mimetype::Text), + (true, false) => Ok(Mimetype::TextCRLF), + (false, true) => Ok(Mimetype::Script), + (true, true) => Ok(Mimetype::ScriptCRLF), + } + } +} + +// https://en.wikipedia.org/wiki/Executable_and_Linkable_Format +fn is_binary(vec: &[u8]) -> Option<Mimetype> { + let binary_re: RegexSet = RegexSet::new(&[ + r"^(?-u)\x7FELF[\x01\x02][\x01\x02]\x01[\x00-\x11]", // Executable and Linkable Format (ELF) + r"^(?-u)\x00\x00\x03\xF3", // AmigaOS loadseg()ble executable/binary + r"^(?-u)MZ", // DOS MZ executable file format and its descendants (including NE and PE) + r"^(?-u)\x64 \x65\x78\x0A\x30\x33\x35\x00", // Dalvik's executable + r"^(?-u)#[!]", // script executable + r"^(?-u)(\xCE|\xCF)\xFA\xED\xFE", // Mach-O binary + r"^(?-u)\x1B\x4C\x75\x61", // Lua bytecode + ]) + .unwrap(); + + if binary_re.is_match(vec) { + return Some(Mimetype::Binary); + } + None +} + +// https://github.com/7h3rAm/cigma/blob/master/cigma/magicbytes.json +// https://en.wikipedia.org/wiki/List_of_file_signatures +fn is_archive(vec: &[u8]) -> Option<Mimetype> { + // we first have to catch zip files with mimetype formats + // - opendocument formats + // - Word Open XML + // Those we do not regard as archives + let special_zip: RegexSet = RegexSet::new(&[ + r"^(?-u)PK\x03\x04.{20,}\x08\x00\x00\x00mimetypeapplication", + r"^(?-u)PK\x03\x04\x14\x00\x06\x00", // Word Open XML (.docx) + r"^(?-u)PK\x03\x04\x14\x00\x08\x00", // Java Jar file + r"^(?-u)PK\x03\x04\x14\x00\x08\x08", // Java Jar file + r"^(?-u)PK\x03\x04\x0A.*?META-INF", // Java Jar file + r"^(?-u)PK\x03\x04.*?META-INF", // Java Jar file + r"^(?-u)PK\x03\x04\x0A.*?\x56\x92\x48\x4F\xEF", // Java Jar file + ]) + .unwrap(); + + if special_zip.is_match(vec) { + return Some(Mimetype::Data); + } + + let archive_re: RegexSet = RegexSet::new(&[ + r"^(?-u)\x37\x7A\xBC\xAF\x27\x1C", // 7zip + r"^(?-u)\x1f\x8B", // gzip (.gz) + r"^(?-u)\x1f\x9D", // LZW (.tar.Z) + r"^(?-u)\x1f\xA0", // LZH (.tar.Z) + r"^(?-u)\xFD\x37\x7A\x58\x5A\x00\x00", // XZ comp. utility using LZMA2 compression (.xz) + r"^(?-u)\x4D\x53\x43\x46", // Microsoft cabinet (.cab) + r"^(?-u)\x42\x5A\x68", // bzip2 + r"^(?-u)\x5A\x57\x53", // lzma + r"^(?-u)\x5D\x00\x00(\x01|\x02|\x04|\x08|\x10|\x20|\x40|\x80)\x00", // lzma + r"^(?-u)\x5D\x00\x00\x00\x01", // lzma + r"^(?-u)(SIT!|SITD|STi0|StuffIt)", // SIT / stuffit (macintosh related) + r"^(?-u)\x4D\x5A", // DOS MZ executable format, but found in zip archives + r"^(?-u)\x52\x61\x72\x21\x1A\x07\x00", // RAR archive version 1.50 onwards + r"^(?-u)\x52\x61\x72\x21\x1A\x07\x01\x00", // RAR archive version 5.0 onwards + // https://en.wikipedia.org/wiki/LHA_(file_format) + r"^(?-u)..-lh[0124567d]", // LHarc (canonical LZH) + r"^(?-u)..-lh[89abce]", // LHarc (Joe Jared extensions) + r"^(?-u)..-lhx", // LHarc (UNLHA32 extensions) + r"^(?-u)..-(pc1|pm0|pm1|pm2|pms)", // LHarc (PMarc extensions) + r"^(?-u)..-lz[s234578]", // LHarc (LArc extensions) + r"^(?-u)\x53\x5a\x44\x44\x88\xf0\x27\x33", // RAR archive version 5.0 onwards + ]) + .unwrap(); + + if archive_re.is_match(vec) { + return Some(Mimetype::Archive); + } + + let archive_re: RegexSet = RegexSet::new(&[ + r"^(?-u)PK(\x03\x04|\x4c\x49\x54\x45|\x30\x30\x50|\x05\x06|\x07\x08)", // zip archive + ]) + .unwrap(); + if archive_re.is_match(vec) { + return Some(Mimetype::Zip); + } + + None +} + +fn analyze_binary(vec: &[u8]) -> Option<Mimetype> { + let rc = is_binary(vec); + if rc.is_some() { + return rc; + } + + let rc = is_archive(vec); + if rc.is_some() { + return rc; + } + + Some(Mimetype::Data) +} + +fn get_filetype(entry: &Path) -> Option<Mimetype> { + match entry.symlink_metadata() { + Ok(mt) => { + let ft = mt.file_type(); + if ft.is_symlink() { + return Some(Mimetype::Symlink); + } + if ft.is_dir() { + return Some(Mimetype::Directory); + } + if ft.is_block_device() { + return Some(Mimetype::BlockDevice); + } + if ft.is_char_device() { + return Some(Mimetype::CharDevice); + } + if ft.is_fifo() { + return Some(Mimetype::Fifo); + } + if ft.is_socket() { + return Some(Mimetype::Socket); + } + None + } + Err(_e) => None, + } +} + +#[test] +fn test_filetype() { + let mut ft = Filetype::new(); + + assert!(ft.analyze("tests_filemagic/zerofile").ok() == Some(Mimetype::Zerofile)); + assert!(ft.analyze("tests_filemagic/a_small_file").ok() == Some(Mimetype::VeryShort)); + assert!(ft.analyze("/dev/null").ok() == Some(Mimetype::CharDevice)); + assert!(ft.analyze("tests_filemagic/").ok() == Some(Mimetype::Directory)); + assert!(ft.analyze("tests_filemagic/zerofile_symlink").ok() == Some(Mimetype::Symlink)); + + assert!(ft.analyze("tests_filemagic/some.pdf").ok() == Some(Mimetype::Pdf)); + + // This file is a pdf but has lines starting with % before the pdf signature shows up + // The unix `file` command) says: data + // analyze() says TextCRLF + //assert!(ft.analyze("tests_filemagic/musterlogo.pdf").ok() == Some(Mimetype::Script)); + + assert!(ft.analyze("tests_filemagic/x.pl").ok() == Some(Mimetype::Script)); + assert!(ft.analyze("tests_filemagic/main.php").ok() == Some(Mimetype::Script)); + + assert!(ft.analyze("tests_filemagic/test.7z").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/x.tgz").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/test.pdf.xz").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/swebib.cab").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/test.tar.bz2").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/PIE.rar").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/infozip-os390.tar.Z").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/bla.lha").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/dvi.zoo").ok() == Some(Mimetype::Archive)); + assert!(ft.analyze("tests_filemagic/rsfs-oztex.sit").ok() == Some(Mimetype::Archive)); + + assert!(ft.analyze("tests_filemagic/empty.zip").ok() == Some(Mimetype::Zip)); + + assert!(ft.analyze("tests_filemagic/README").ok() == Some(Mimetype::Text)); + // assert!(ft.analyze("tests_filemagic/README1").ok() == Some(Mimetype::Text)); + + assert!(ft.analyze("tests_filemagic/cp").ok() == Some(Mimetype::Binary)); + assert!(ft.analyze("tests_filemagic/cheq-f.sit-hqx").ok() == Some(Mimetype::Binary)); + assert!(ft.analyze("tests_filemagic/MuchMore").ok() == Some(Mimetype::Binary)); + + assert!(ft.analyze("tests_filemagic/support.ps").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/rosette.eps").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/eutest.ps").ok() == Some(Mimetype::Data)); + // assert!(ft.analyze("tests_filemagic/NORMAL.PS").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/chap5.rtf").ok() == Some(Mimetype::Data)); + assert!(ft.analyze("tests_filemagic/commons-math.jar").ok() == Some(Mimetype::Data)); + + assert!(ft.analyze("tests_filemagic/8stbu11h.htm").ok() == Some(Mimetype::TextCRLF)); +} diff --git a/support/pkgcheck/src/generate.pest b/support/pkgcheck/src/generate.pest new file mode 100644 index 0000000000..749cb60e5e --- /dev/null +++ b/support/pkgcheck/src/generate.pest @@ -0,0 +1,34 @@ +file = { SOI ~ ( generatefile_stmt | generate_stmt | filecontents | other)* ~ EOI } + +other = _{ (!NEWLINE ~ ANY) } + +generatefile_stmt = _{ "\\generateFile" ~ LPAREN ~ filename ~ RPAREN } + +generate_stmt = _{ "\\generate" ~ LPAREN ~ (file_stmt | cmd )+ ~ RPAREN } +file_stmt = _{ "\\file" ~ LPAREN ~ filename ~ RPAREN ~ + LPAREN ~ (from | cmd) + ~RPAREN } + +filecontents = _{ "\\begin" ~ LPAREN ~ filecontents_lit ~ RPAREN ~ + LPAREN ~ filename ~ RPAREN } + +filecontents_lit = _{ "filecontents*" | "filecontents" } + +filename = { (!RPAREN ~ ANY)* } + +from = _{ "\\from" ~ LPAREN ~ anyx ~ RPAREN ~ + LPAREN ~ anyx ~ RPAREN } + +cmd = _{ "\\" ~ ident ~ (LPAREN ~ anyx ~ RPAREN)* } + +ident_char = _{ 'a'..'z' | 'A'..'Z' | '0'..'9' | "_" } +ident = _{ + ('a'..'z' | 'A'..'Z') ~ ident_char* | + "_" ~ ident_char+ } + + +LPAREN = _{ "{" } +RPAREN = _{ "}" } + +anyx = _{ (!RPAREN ~ ANY)* } +WHITESPACE = _{ " " | "\t" | NEWLINE } +COMMENT = _{ "%" ~ (!NEWLINE ~ ANY)* }
\ No newline at end of file diff --git a/support/pkgcheck/src/generate.pest.md b/support/pkgcheck/src/generate.pest.md new file mode 100644 index 0000000000..643fe57750 --- /dev/null +++ b/support/pkgcheck/src/generate.pest.md @@ -0,0 +1,15 @@ +# Some notes about gparser.pest + +## COMMENT and WHITESPACE + +The Pest book at https://pest.rs/book/grammars/syntax.html says: + +> Many languages and text formats allow arbitrary whitespace and comments between logical tokens. +> For instance, Rust considers 4+5 equivalent to 4 + 5 and 4 /* comment */ + 5. + +> The optional rules WHITESPACE and COMMENT implement this behaviour. If either (or both) are defined, +> they will be implicitly inserted at every sequence and between every repetition (except in atomic rules). + + +Therefore COMMENT and WHITESPACE are defined in `gparser.pest`. + diff --git a/support/pkgcheck/src/gparser.rs b/support/pkgcheck/src/gparser.rs new file mode 100644 index 0000000000..6f9d7f393a --- /dev/null +++ b/support/pkgcheck/src/gparser.rs @@ -0,0 +1,46 @@ +use pest::Parser; + +#[cfg(debug_assertions)] +const _GRAMMAR: &'static str = include_str!("generate.pest"); + +#[derive(Parser)] +#[grammar = "generate.pest"] +pub struct GenerateParser; + +pub fn parse_generate(s: &str) -> Option<Vec<String>> { + let pairs = GenerateParser::parse(Rule::file, s); + + if pairs.is_err() { + return None; + } + + let mut found = Vec::new(); + + for pair in pairs.unwrap() { + // A pair is a combination of the rule which matched and a span of input + + // A pair can be converted to an iterator of the tokens which make it up: + for inner_pair in pair.into_inner() { + let inner_span = inner_pair.clone().as_span(); + match inner_pair.as_rule() { + Rule::filename => { + //println!("fname: {}", inner_span.as_str()); + // https://rust-lang-nursery.github.io/rust-clippy/v0.0.212/index.html#clone_double_ref + let s: String = inner_span.as_str().to_string(); + //println!("filename: {}", s); + found.push(s); + } + Rule::EOI => (), + e => { + println!("unreachable: {:?}", e); + unreachable!() + } + }; + } + } + if found.is_empty() { + None + } else { + Some(found) + } +} diff --git a/support/pkgcheck/src/linkcheck.rs b/support/pkgcheck/src/linkcheck.rs new file mode 100644 index 0000000000..60b993e960 --- /dev/null +++ b/support/pkgcheck/src/linkcheck.rs @@ -0,0 +1,259 @@ +use threadpool::ThreadPool; + +use colored::*; +use linkify::{LinkFinder, LinkKind}; + +use std::fs::File; +use std::io::prelude::*; + +//use reqwest::Client; +use openssl_probe; +use reqwest; +use reqwest::header; +use std::time::Duration; +//use std::thread; +use std::sync::Arc; +use std::sync::Mutex; + +use std::sync::atomic::Ordering; + +use std::collections::HashMap; +use std::collections::HashSet; + +enum UrlStatus { + Unknown, + UrlOk, + UrlError(String), +} + +struct HashVal { + paths: HashSet<String>, + status: UrlStatus, +} + +type UrlHash = HashMap<String, HashVal>; + +pub struct LinkCheck { + pool: Mutex<ThreadPool>, + urlhash: Arc<Mutex<UrlHash>>, + print_all: bool, +} + +impl LinkCheck { + pub fn new(num_threads: usize, print_all: bool) -> LinkCheck { + openssl_probe::init_ssl_cert_env_vars(); + let pool = Mutex::new(ThreadPool::new(num_threads)); + LinkCheck { + pool, + urlhash: Arc::new(Mutex::new(HashMap::default())), + print_all, + } + } + + pub fn check_urls(&self, fname: &str) { + let print_all = self.print_all; + if let Some(links) = get_links(fname) { + for l in links { + let urlhash = self.urlhash.clone(); + let fname_s = String::from(fname); + self.pool.lock().unwrap().execute(move || { + check_link(&l, &fname_s, &urlhash, print_all); + }); + } + } + } +} + +impl Drop for LinkCheck { + fn drop(&mut self) { + //println!("Now dropping ..."); + let pool = self.pool.lock().unwrap(); + pool.join(); + } +} + +fn check_link(url: &str, fname: &str, urlhash: &Arc<Mutex<UrlHash>>, print_all: bool) { + let url = String::from(url); + + let mut run_check_link = false; + + // It is very important to keep the lock for the urlhash + // only for a short period of time + // + // If we don't find the url in the urlhash then + // we set `run_check_link` to `true` so that we will + // check the url + { + let f = String::from(fname); + + let mut urlhash = urlhash.lock().unwrap(); + if !urlhash.contains_key(&url) { + let mut hs = HashSet::default(); + hs.insert(f.clone()); + let url1 = url.clone(); + + urlhash.insert( + url1, + HashVal { + status: UrlStatus::Unknown, + paths: hs, + }, + ); + run_check_link = true; + } else if let Some(hs) = urlhash.get_mut(&url) { + match &hs.status { + UrlStatus::Unknown => { + hs.paths.insert(f); + } + UrlStatus::UrlOk => { + if print_all { + print_ok(no_colors!(), &url, &f); + }; + } + UrlStatus::UrlError(e) => { + e0022!(f, e); + } + } + } + } + + // + if run_check_link { + match check_link_inner(&url, true) { + UrlStatus::UrlOk => { + let mut urlhash = urlhash.lock().unwrap(); + if let Some(mut hs) = urlhash.get_mut(&url) { + if print_all { + for p in hs.paths.iter() { + print_ok(no_colors!(), &url, p); + } + } + hs.status = UrlStatus::UrlOk; + } + } + UrlStatus::UrlError(e) => { + let mut urlhash = urlhash.lock().unwrap(); + if let Some(mut hs) = urlhash.get_mut(&url) { + for p in hs.paths.iter() { + e0022!(p, e); + } + hs.status = UrlStatus::UrlError(e); + } + } + _ => (), + } + } +} + +fn get_links(fname: &str) -> Option<Vec<String>> { + let fhdl = File::open(fname); + match fhdl { + Ok(mut f) => { + let mut buf = Vec::new(); + + match f.read_to_end(&mut buf) { + Ok(_bytes_read) => { + return get_links_inner(&String::from_utf8_lossy(&buf)); + } + Err(e) => println!("Error reading file {}: {:?}", fname, e), + } + } + Err(e) => println!("Error opening file {}: {}", fname, e), + } + + None +} + +// retrieves links in a string and then checks those links +fn get_links_inner(s: &str) -> Option<Vec<String>> { + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]); + let links: Vec<_> = finder.links(s).collect(); + let result: Vec<&str> = links.iter().map(|e| e.as_str()).collect(); + + let mut links = vec![]; + for r in result { + if !r.starts_with("http://") && !r.starts_with("https://") && !r.starts_with("ftp://") { + continue; + } + // This is a workaround to prevent URLs ending with ` + if r.ends_with('`') { + links.push(String::from(&r[..r.len()-1])); + } else { + links.push(String::from(r)); + } + + } + if !links.is_empty() { + Some(links) + } else { + None + } +} + +fn check_link_inner(l: &str, head: bool) -> UrlStatus { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::USER_AGENT, + header::HeaderValue::from_static( + "Mozilla/5.0 (X11; Linux i686; rv:64.0) Gecko/20100101 Firefox/64.0", + ), + ); + + let default_policy = reqwest::RedirectPolicy::default(); + let policy = reqwest::RedirectPolicy::custom(move |attempt| { + if attempt.url().host_str() == Some("127.0.0.1") { + attempt.stop() + } else { + default_policy.redirect(attempt) + } + }); + + let cb = reqwest::Client::builder() + .gzip(true) + .redirect(policy) + .default_headers(headers) + .timeout(Duration::from_secs(7)) + .build() + .unwrap(); + // let url: Url = + // match l.parse() { + // Ok(url) => url, + // Err(e) => { println!("Error: {:?}", e); panic!("Scheiss"); } + // }; + let resp = if head { + cb.head(l).send() + } else { + cb.get(l).send() + }; + match resp { + Ok(s) => { + if s.status().is_informational() + || s.status().is_success() + || s.status().is_redirection() + { + return UrlStatus::UrlOk; + } + + if head { + check_link_inner(l, false) + } else { + let e = format!("{}: {}", l, s.status()); + UrlStatus::UrlError(e) + } + } + Err(e) => { + let e = format!("{}", e); + UrlStatus::UrlError(e) + } + } +} + +fn print_ok(no_colors: bool, url: &str, f: &str) { + if no_colors { + println!("✔ {} in {}", &url, f); + } else { + // println!("✔ {} in {}", &url, f); + println!("{} {} in {}", "✔".bright_green().bold(), url, f); + } +} diff --git a/support/pkgcheck/src/main.rs b/support/pkgcheck/src/main.rs new file mode 100644 index 0000000000..a7be277330 --- /dev/null +++ b/support/pkgcheck/src/main.rs @@ -0,0 +1,1287 @@ +extern crate pest; +#[macro_use] +extern crate pest_derive; + +mod gparser; + +#[macro_use] +mod messages; +use messages::{explains, explains_all}; + +#[macro_use] +mod utils; +mod linkcheck; + +mod recode; +use recode::crlf2lf; + +mod filemagic; + +use linkcheck::LinkCheck; +use std::fmt; +use std::fmt::Display; +use std::str; +use utils::*; + +use std::os::unix::fs::MetadataExt; + +use scoped_threadpool::Pool; + + +use tempdir::TempDir; +#[macro_use] +extern crate lazy_static; +use colored::*; + +use regex::Regex; + +use std::fs::File; +use std::fs::Metadata; +use std::io::prelude::*; +use std::io::BufReader; +use std::os::unix::fs::FileTypeExt; +use std::{fs, process}; +// use std::os::unix::fs::PermissionsExt; + +use std::ffi::OsStr; +use std::io::{self, Read}; +use std::path::Path; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; + +use blake2::{Blake2b, Digest}; +use fnv::{FnvHashMap as HashMap, FnvHashSet as HashSet}; +use std::sync::mpsc::{channel, Sender}; +use structopt::StructOpt; +#[cfg(unix)] +use walkdir::{DirEntry, WalkDir}; + +fn err(path: &PathBuf, err: &io::Error) { + e0027!(path.display(), err); +} + +type HashSender = Sender<(u64, PathBuf, Vec<u8>)>; +//type DupeSender = Sender<(u64, Vec<PathBuf>)>; + +const BLOCKSIZE: usize = 4096; + +fn hash_file_inner(path: &PathBuf) -> io::Result<Vec<u8>> { + let mut buf = [0u8; BLOCKSIZE]; + let mut fp = File::open(&path)?; + let mut digest = Blake2b::default(); + // When we compare byte-by-byte, we don't need to hash the whole file. + // Instead, hash a block of 4kB, skipping 100kB. + loop { + match fp.read(&mut buf)? { + 0 => break, + n => digest.input(&buf[..n]), + } + } + Ok(digest.result().to_vec()) +} + +fn hash_file(fsize: u64, path: PathBuf, tx: &HashSender) { + match hash_file_inner(&path) { + Ok(hash) => tx.send((fsize, path, hash)).unwrap(), + Err(e) => err(&path, &e), + } +} + +fn fix_perms(entry: &str) { + i0005!(entry); + let rc = run_cmd("/bin/chmod", &["-v", "ug=rwX,o=rX", entry]); + + if rc.status { + if let Some(op) = rc.output { + print!("{}", op); + } + } +} + +fn filename_has_bad_chars(entry: &DirEntry) -> Option<(usize, char)> { + let s = entry.file_name().to_str().unwrap(); + + for (i, c) in s.char_indices() { + match c { + '\x00'...'\x1f' => return Some((i, c)), + ' ' | '!' | '"' | '#' | '$' | '%' | '&' | '\'' => return Some((i, c)), + '(' | ')' | '*' => return Some((i, c)), + '+' | ',' | '-' | '.' => (), + '0'...'9' => (), + ':' | ';' | '<' | '=' | '>' | '?' | '@' => return Some((i, c)), + 'A'...'Z' => (), + '[' | '\\' | ']' | '^' => return Some((i, c)), + '_' => (), + '`' => return Some((i, c)), + 'a'...'z' => (), + '{' | '|' | '}' | '~' => return Some((i, c)), + '/' => return None, + _ => return Some((i, c)), + }; + } + None +} + +// returns false if CRLF, otherwise true +fn fix_crlf(fname: &str) -> bool { + i0004!(fname); + //run_cmd("/usr/bin/fromdos", &["-p", "-e", "-v", fname]).status + match crlf2lf(fname) { + Ok(_) => { + i0007!(fname); + true + } + Err(e) => { + e0027!(fname, e); + false + } + } +} + +fn check_readme(dir_entry: &str, is_readme: &ReadmeKind, ft: &filemagic::Mimetype) -> bool { + let msg_name = if let ReadmeKind::Symlink(s) = is_readme { + format!("{} (symlinked from {})", dir_entry, &s) + } else { + dir_entry.to_string() + }; + + match ft { + filemagic::Mimetype::Pdf + | filemagic::Mimetype::Binary + | filemagic::Mimetype::Archive + | filemagic::Mimetype::Zip => { + e0003!(msg_name); + return false; + } + filemagic::Mimetype::BOM(b) => { + e0029!(msg_name, b.as_ref()); + return false; + } + filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => match File::open(dir_entry) { + Ok(f) => { + if !check_readme_inner(&msg_name, &f) { + return false; + } + } + Err(e) => { + e0027!(msg_name, e); + return false; + } + }, + _ => (), + } + true +} + +fn check_readme_inner(fname: &str, f: &std::fs::File) -> bool { + let reader = BufReader::new(f); + + let lines = reader.split(b'\n').map(|l| l.unwrap()); + let mut result = true; + + for (lineno, line) in lines.enumerate() { + if let Err(e) = String::from_utf8(line.clone()) { + e0021!(fname, lineno + 1, e); + result = false; + } + } + result +} + +fn is_readme(entry: &DirEntry) -> bool { + let r = entry.file_name().to_str().unwrap(); + + match r { + "README" | "README.txt" | "README.md" => true, + _ => false, + } +} + +fn get_devno(meta: &Metadata) -> u64 { + meta.dev() +} + +fn _get_devno(entry: &DirEntry) -> u64 { + let meta = fs::metadata(entry.path().to_str().unwrap()); + match meta { + Ok(m) => m.dev(), + _ => 0, + } +} + +#[derive(StructOpt, Debug)] +#[structopt(about = "A checker for uploaded packages to CTAN.")] +struct Args { + #[structopt(short = "I", long = "ignore-dupes", help = "Ignore dupes")] + ignore_dupes: bool, + #[structopt(short = "v", long = "verbose", help = "Verbose operation?")] + verbose: bool, + #[structopt(short = "L", long = "correct-crlf", help = "Correct CRLF line endings")] + correct_crlf: bool, + #[structopt(short = "C", long = "correct-perms", help = "Correct permissions")] + correct_perms: bool, + #[structopt(long = "no-colors", help = "Don't display messages in color")] + no_colors: bool, + #[structopt(long = "urlcheck", help = "Check URLs found in README files")] + urlcheck: bool, + #[structopt(short = "T", long = "tds-zip", help = "tds zip archive", group = "tds")] + tds_zip: Option<String>, + #[structopt( + short = "e", + long = "explain", + help = "Explain error or warning message", + group = "only_one" + )] + explain: Option<String>, + #[structopt( + long = "explain-all", + help = "Explains all error or warning messages", + group = "only_one" + )] + explain_all: bool, + #[structopt( + long = "show-temp-endings", + help = "Show file endings for temporary files", + group = "only_one" + )] + show_tmp_endings: bool, + #[structopt(short = "d", long = "package-dir", help = "Package directory")] + pkg_dir: Option<String>, +} + +// We take care to avoid visiting a single inode twice, +// which takes care of (false positive) hardlinks. +#[cfg(unix)] +fn check_inode(set: &mut HashSet<(u64, u64)>, meta: &Metadata) -> bool { + set.insert((get_devno(meta), meta.ino())) +} + +#[cfg(not(unix))] +fn check_inode(_: &mut HashSet<u64>, _: &Metadata) -> bool { + true +} + +lazy_static! { + static ref ARGS: Args = Args::from_args(); + static ref ERROR_OCCURED: AtomicBool = AtomicBool::new(false); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DPath { + Both(PathBuf), + Tds(PathBuf), +} + +impl fmt::Display for DPath { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + DPath::Both(p) => write!(f, "{}", p.display()), + DPath::Tds(p) => write!(f, "{}", p.display()), + } + } +} + +#[derive(Default)] +pub struct DupPath { + len: usize, + plen: usize, + dupes: Vec<DPath>, +} + +impl DupPath { + pub fn new() -> DupPath { + DupPath { + len: 0, + plen: 0, + dupes: Vec::new(), + } + } + + pub fn push(&mut self, pb: PathBuf) { + let pname = pb.to_string_lossy(); + self.len += 1; + if pname.ends_with(".tfm") { + self.dupes.push(DPath::Tds(pb.clone())); + } else { + self.plen += 1; + self.dupes.push(DPath::Both(pb.clone())); + } + } +} + +type Hashes = HashMap<(u64, Vec<u8>), DupPath>; + +fn main() { + match &ARGS.explain { + None => (), + Some(e) => { + explains(&e); + process::exit(0); + } + } + + if ARGS.explain_all { + explains_all(); + process::exit(0); + } + + if ARGS.show_tmp_endings { + show_tmp_endings(); + process::exit(0); + } + + let pkg_dir = match &ARGS.pkg_dir { + None => { + f0001!(); + process::exit(1); + } + Some(d) => { + if !exists_dir(&d) { + f0002!(d); + process::exit(1); + }; + d + } + }; + + let tds_zip = &ARGS.tds_zip; + let pkg_name = match tds_zip { + None => None, + Some(tz) => { + let pn = check_tds_archive_name(tds_zip); + if !exists_file(&tz) { + f0003!(&tz); + process::exit(1); + } + let mut fmagic = filemagic::Filetype::new(); + match fmagic.analyze(&tz) { + Ok(filemagic::Mimetype::Zip) => (), + _ => { + f0004!(&tz); + process::exit(1) + } + }; + pn + } + }; + + if let Some(hashes) = check_package(pkg_dir, tds_zip) { + if tds_zip.is_some() { + if let Some(pn) = pkg_name { + if let Some(s) = ARGS.tds_zip.as_ref() { + match check_tds_archive(&pn, s, &hashes) { + Ok(_) => (), + Err(e) => { + e0027!(s, e); + } + } + } + } + } + } + + if ERROR_OCCURED.load(Ordering::Relaxed) { + process::exit(1); + } else { + process::exit(0); + } +} + +fn print_duplicates(hashes: &Hashes) { + let mut total_dupes = 0; + let mut total_files = 0; + let mut total_size = 0; + + let mut header_printed = false; + for (k, paths) in hashes.iter() { + let (sz, _hash) = k; + + if paths.plen <= 1 { + total_files += 1; + total_size += sz; + continue; + } else if !header_printed { + w0002!(); + header_printed = true; + } + + total_files += paths.plen; + total_size += sz * (paths.plen - 1) as u64; + total_dupes += (paths.plen - 1) as u64; + + println!("Size: {}", sz); + for p in &paths.dupes { + if let DPath::Both(p) = p { + let ps = p.as_path().to_str().unwrap(); + println!(" >>> {}", ps); + } + } + println!(); + } + + if ARGS.verbose && total_dupes > 0 { + println!("Duplicate statistics"); + println!(" Found {} duplicate files", total_files); + println!(" Size of duplicate files: {}", total_size); + } +} + +//#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] +pub enum FType { + Regular, + Directory, + Symlink, + BlockDevice, + CharDevice, + Fifo, + Socket, + Error(String), +} + +fn get_filetype(entry: &DirEntry) -> FType { + match entry.metadata() { + Ok(mt) => { + let ft = mt.file_type(); + if ft.is_symlink() { + return FType::Symlink; + } + if ft.is_dir() { + return FType::Directory; + } + if ft.is_block_device() { + return FType::BlockDevice; + } + if ft.is_char_device() { + return FType::CharDevice; + } + if ft.is_fifo() { + return FType::Fifo; + } + if ft.is_socket() { + return FType::Socket; + } + FType::Regular + } + Err(e) => FType::Error(format!("{}", e)), + } +} + +// +// read file into buffer[u8] +// then convert into string +// +fn check_generated_files(entry: &str, generated: &mut HashMap<PathBuf, PathBuf>) { + // unwrap() is ok here as we only call this function for files, + // specifically .ins or .dtx files + let entry_fname = filename(entry).unwrap().to_string(); + let entry_base = &entry_fname[0..entry_fname.len() - 4]; + + let fhdl = File::open(&entry); + match fhdl { + Ok(mut f) => { + let mut buf = Vec::new(); + + match f.read_to_end(&mut buf) { + Ok(_bytes_read) => { + if let Some(found) = + gparser::parse_generate(&String::from_utf8_lossy(&buf.clone())) + { + for fname in found { + // As we request a README in the top level directory of + // a package we ignore if a README was generated by an + // .ins or .dtx file + // CAVEAT: If this happens in a subdirectory it could be an error!!!! + if fname == "README" || fname == "README.txt" || fname == "README.md" { + continue; + } + + // If the filename in the 'file{} statement is jobname.<ext> + // we replace jobname with the filename before we investigate further + let fname_x = if fname.starts_with("\\jobname") { + let ext = match get_extension_from_filename(&fname) { + Some(e) => e, + None => "", + }; + format!("{}.{}", entry_base, ext) + } else { + fname.to_string() + }; + + let pb = PathBuf::from(fname_x); + + generated.entry(pb).or_insert_with(|| PathBuf::from(entry)); + } + }; + } + Err(e) => println!("Error reading file {}: {:?}", entry, e), + } + } + Err(e) => println!("Error opening file {}: {:?}", entry, e), + } + return; +} + +fn x_bit_set(p: u32) -> bool { + let p1 = p & 0o7777; + p1 & 0o111 != 0 +} + +// checks that archive name is in the format x.tds.zip and returns: +// None if not +// Option<x> if x.tds.zip +// Caveat: we currently don't know if x is really the package name. +fn check_tds_archive_name(tds_zip: &Option<String>) -> Option<String> { + match tds_zip { + None => None, + Some(tz) => { + // 8 for ".tds.zip", 1 means a package name is at least a character + if tz.len() < 8 + 1 || !tz.ends_with(".tds.zip") { + f0005!(tz); + process::exit(1); + } + let mut pname = String::from(utils::basename(&tz)); + let plen = pname.len(); + pname.truncate(plen - 8); + Some(pname.to_string()) + } + } +} + +fn unzip_tds_archive(tds_zip: &str, tmp_dir: &str) { + match run_cmd("/usr/bin/unzip", &["-q", "-d", tmp_dir, &tds_zip]) { + CmdReturn { status: true, .. } => (), + CmdReturn { + status: false, + output: out, + } => { + if let Some(o) = out { + e0033!(&tds_zip, o); + } else { + e0033!(&tds_zip, "<no error given"); + } + process::exit(1); + } + } +} + +fn check_tds_archive( + pkg_name: &str, + tds_zip: &str, + hashes: &HashMap<(u64, Vec<u8>), DupPath>, +) -> Result<(), io::Error> { + i0003!(tds_zip); + + let dir_entry = Path::new(tds_zip); + let p = get_perms(&dir_entry); + if !owner_has(p, 4) || !others_have(p, 4) || x_bit_set(p) { + e0024!(tds_zip, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&tds_zip); + set_perms(&tds_zip, 0o664); + } + }; + + let tmp_dir = TempDir::new("pkgcheck")?; + let tmp_dir_offset = tmp_dir.path().to_str().unwrap().len() + 1; + unzip_tds_archive(tds_zip, tmp_dir.path().to_str().unwrap()); + + let tds_toplevel_dirs: HashSet<String> = [ + "tex", "fonts", "metafont", "metapost", "bibtex", "scripts", "doc", "source", + ] + .iter() + .map(|&s| s.to_string()) + .collect(); + + for entry in fs::read_dir(tmp_dir.path())? { + let entry = entry?; + let path = entry.path(); + + match path.file_stem() { + None => (), + Some(file_stem) => { + let fs = file_stem.to_str().expect("bad file name"); + if !tds_toplevel_dirs.contains(fs) { + if path.is_dir() { + e0020!(fs); + } else { + e0034!(fs); + } + } + } + } + + // if path.is_dir() { + // match path.file_stem() { + // None => (), + // Some(file_stem) => { + // let fs = file_stem.to_str().expect("bad file name"); + // if !tds_toplevel_dirs.contains(fs) { + // e0020!(fs); + // } + // } + // } + // } else { + // println!("mist"); + // e0020!(path.display()); + // } + } + + // in order to compare the package files with the content of the + // tds zip archive we need to checksum the files in the tds zip + // archive. + + let mut sizes = HashMap::default(); + let mut pool = Pool::new(num_cpus::get() as u32 + 1); + { + // Processing a single file entry, with the "sizes" hashmap collecting + // same-size files. Entries are either Found::One or Found::Multiple, + // so that we can submit the first file's path as a hashing job when the + // first duplicate is found. Hashing each file is submitted as a job to + // the pool. + let mut process = |fsize, dir_entry: &DirEntry| { + let path = dir_entry.path().to_path_buf(); + let sizeref = &mut sizes; + + sizeref.entry(fsize).or_insert_with(Vec::new).push(path); + }; + + let mut map_files_found = false; + let mut map_dvips_found = false; + + for dir_entry in WalkDir::new(tmp_dir.path().to_str().unwrap()).follow_links(false) { + match dir_entry { + Ok(dir_entry) => { + //let dir_entry_str = dir_entry.path().to_str().unwrap(); + + let ft = get_filetype(&dir_entry); + + if ft == FType::Directory { + continue; + } + + if let FType::Error(e) = ft { + e0023!(e); + continue; + } + + match dir_entry.metadata() { + Ok(meta) => { + let fsize = meta.len(); + process(fsize, &dir_entry); + + let dir_entry_str = match dir_entry.path().to_str() { + Some(d) => d, + None => { + e0031!(dir_entry.path().to_string_lossy()); + continue; + } + }; + let dir_entry_display = &dir_entry_str[tmp_dir_offset..]; + + if is_temporary_file(&dir_entry_str) { + e0008!(dir_entry_display); + } + + // if the path doesn't contain a man page... + if dir_entry_str.find("/man/").is_none() { + let pkg_name_s = format!("/{}/", pkg_name); + // ...then we want to have the package name in the path + if dir_entry_str.find(&pkg_name_s).is_none() { + e0028!(pkg_name, dir_entry_display); + } + } + + if dir_entry_str.ends_with(".map") { + map_files_found = true; + let re: Regex = Regex::new(r"fonts[/]map[/]dvips[/]").unwrap(); + if re.is_match(dir_entry_str) { + map_dvips_found = true; + } + + // if ( $fname =~ m{ fonts [/] map [/] dvips [/] }xms ) { + // $map_dvips_found = 1; + // } + } + } + Err(e) => { + e0027!(dir_entry.path().display(), e); + } + } + } + Err(e) => { + eprintln!("{}", e); + } + } + } + + if map_files_found && !map_dvips_found { + println!( "Map files found for package {} but none of them is in a path starting with fonts/map/dvips", pkg_name); + } + }; + + let mut tds_hashes: HashMap<(u64, Vec<u8>), Vec<PathBuf>> = HashMap::default(); + pool.scoped(|scope| { + let (tx, rx) = channel(); + + let hashref = &mut tds_hashes; + scope.execute(move || { + for (size, path, hash) in rx.iter() { + hashref + .entry((size, hash)) + .or_insert_with(Vec::new) + .push(path); + } + }); + + for size in sizes.keys() { + for p in &sizes[size] { + let txc = tx.clone(); + scope.execute(move || hash_file(*size, p.to_path_buf(), &txc)); + } + } + }); + + tmp_dir.close()?; + // now check if each package file is in the tds archive + for (k, paths) in hashes.iter() { + if !tds_hashes.contains_key(k) { + let p = &paths.dupes[0]; + e0026!(p); + } + } + + Ok(()) +} + +fn get_extension_from_filename(filename: &str) -> Option<&str> { + Path::new(filename).extension().and_then(OsStr::to_str) +} + +fn found_unwanted_filetype(fname: &str, ft: &FType) -> bool { + match ft { + FType::Socket => { + e0013!(fname); + true + } + FType::Fifo => { + e0014!(fname); + true + } + FType::BlockDevice => { + e0015!(fname); + true + } + FType::CharDevice => { + e0016!(fname); + true + } + FType::Error(e) => { + e0023!(e); + true + } + _ => false, + } +} + +fn check_and_correct_perms(dir_entry: &str, p: u32) { + if owner_has(p, 5) || !others_have(p, 4) { + e0002!(dir_entry, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry); + set_perms(&dir_entry, 0o664); + } + }; +} + + +#[derive(Debug, Clone, PartialEq)] +enum FileKind { + File, + Directory, + Symlink(String), +} + + +impl Display for FileKind { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { + match *self { + FileKind::File => f.write_str("file"), + FileKind::Directory => f.write_str("directory"), + FileKind::Symlink(_) => f.write_str("symlink"), + } + } +} + + + +#[derive(Debug, Clone, PartialEq)] +enum ReadmeKind { + No, + Yes, + Symlink(String), +} + +fn check_package(root: &str, tds_zip: &Option<String>) -> Option<Hashes> { + //let mut urls = HashMap::default(); //: <String,UrlStatus> + + let mut lcnames: HashMap<PathBuf, Vec<(PathBuf, FileKind)>> = HashMap::default(); + + let mut doublenames: HashMap<PathBuf, Vec<PathBuf>> = HashMap::default(); + + let mut inodes = HashSet::default(); + + i0002!(root); + // This hash contains all package file names. + // + // PathBuf: the full path starting at the directory specified at the command line + // Metadata: the meta data of the file + // String: the file name without any directory part + // ReadmeKind: is it a certain README, file or symlink? + // A special case of a README file is a file with has a different name but + // was poinded to by a symlink. Example: README --> README.rst + let mut file_names: HashMap<PathBuf, (Metadata, String, ReadmeKind)> = HashMap::default(); + { + let mut readme_found = false; + + let root_absolute = PathBuf::from(root) + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(); + for dir_entry in WalkDir::new(root).follow_links(false) { + match dir_entry { + Ok(dir_entry) => { + let dir_entry_str = match dir_entry.path().to_str() { + Some(d) => d, + None => { + e0031!(dir_entry.path().to_string_lossy()); + continue; + } + }; + let meta = match dir_entry.metadata() { + Ok(meta) => meta, + Err(e) => { + e0023!(e); + continue; + } + }; + + if !check_inode(&mut inodes, &meta) { + continue; + } + + // this is the file_name without the directory part + // unwrap() is ok here as we covered potential UTF-8 related errors + // above in the definition of dir_entry_str + let file_name = dir_entry.file_name().to_str().unwrap().to_string(); + + let ft = get_filetype(&dir_entry); + + if found_unwanted_filetype(dir_entry_str, &ft) { + continue; + } + + if let Some((offset, ch)) = filename_has_bad_chars(&dir_entry) { + e0001!(ch, dir_entry_str, offset); + } + + // 1. dealing with symlinks + if ft == FType::Symlink { + match get_symlink(&dir_entry) { + Ok(None) => { + e0010!(dir_entry_str); + continue; + } + Err(e) => { + e0027!(dir_entry_str, e); + continue; + } + Ok(Some(p)) => { + let pd: String = + p.canonicalize().unwrap().to_string_lossy().to_string(); + if !pd.starts_with(&root_absolute) { + e0030!(dir_entry_str, p.display()); + continue; + } + + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + if let Some(_dir_name) = filename(dir_entry_str) { + let lcnref = &mut lcnames; + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push(( + PathBuf::from(dir_entry_str), + //FileKind::Symlink(dir_entry_str.into()), + FileKind::Symlink(pd.clone()), + )); + } + if is_readme(&dir_entry) { + readme_found = true; + file_names.insert( + p, + ( + meta, + file_name, + ReadmeKind::Symlink(dir_entry_str.to_string()), + ), + ); + } + continue; + } + } + } + + let p = get_perms(&dir_entry.path()); + + // 2. dealing with directories + if ft == FType::Directory { + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + if let Some(_dir_name) = filename(dir_entry_str) { + let lcnref = &mut lcnames; + + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push((PathBuf::from(dir_entry_str), FileKind::Directory)); + } + + if !owner_has(p, 5) || !others_have(p, 5) { + e0011!(dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry_str); + set_perms(&dir_entry_str, 0o775); + } + } + + match is_empty_directory(&dir_entry) { + Ok(true) => { + e0004!(dir_entry_str); + } + Err(e) => { + e0027!(dir_entry_str, e); + } + Ok(false) => (), + } + if is_hidden_directory(&dir_entry) { + e0006!(dir_entry_str); + } + + if is_unwanted_directory(&dir_entry.file_name().to_str().unwrap()) { + e0018!(dir_entry_str); + } + continue; + } + + // 3. dealing with regular files + if is_hidden(&file_name) { + e0007!(dir_entry_str); + } + if is_temporary_file(&dir_entry_str) { + e0008!(dir_entry_str); + } + + + if let Some(file_name) = filename(dir_entry_str) { + let doubleref = &mut doublenames; + + doubleref + .entry(PathBuf::from(file_name)) + .or_insert_with(Vec::new) + .push(PathBuf::from(dir_entry_str)); + } + + + if is_readme(&dir_entry) { + // We want to deal with README files only if they are + // in the root directory of the package. + let f = format!( + "{}{}{}", + root, + // we have to pay attention if `root` ends already with '/' + if root.ends_with('/') { "" } else { "/" }, + &file_name + ); + + if dir_entry_str == f { + readme_found = true; + file_names.insert( + dir_entry.path().to_path_buf(), + (meta, file_name.clone(), ReadmeKind::Yes), + ); + } else { + file_names.entry(dir_entry.path().to_path_buf()).or_insert(( + meta, + file_name.clone(), + ReadmeKind::No, + )); + } + } else { + file_names.entry(dir_entry.path().to_path_buf()).or_insert(( + meta, + file_name.clone(), + ReadmeKind::No, + )); + } + + let lc_dir_entry_str = dir_entry_str.to_lowercase(); + + let lcnref = &mut lcnames; + + lcnref + .entry(PathBuf::from(lc_dir_entry_str)) + .or_insert_with(Vec::new) + .push((PathBuf::from(dir_entry_str), FileKind::File)); + + } + + Err(e) => { + eprintln!("{}", e); + } + } + } + + if !readme_found { + e0009!(); + } + } + + let lc = LinkCheck::new(4, false); + let mut detective = filemagic::Filetype::new(); + + let mut sizes = HashMap::default(); + let mut generated: HashMap<PathBuf, PathBuf> = HashMap::default(); + + { + // Processing a single file entry, with the "sizes" hashmap collecting + // same-size files. Entries are either Found::One or Found::Multiple, + // so that we can submit the first file's path as a hashing job when the + // first duplicate is found. Hashing each file is submitted as a job to + // the pool. + let mut process = |fsize, path: &PathBuf| { + let sizeref = &mut sizes; + + let path = path.clone(); + sizeref.entry(fsize).or_insert_with(Vec::new).push(path); + }; + for (path, (meta, _file_name, is_readme)) in file_names.iter() { + + //println!("DEBUG: path {:?}, file_name {:?}, is_readme: {} ", &path, &file_name, is_readme); + let dir_entry_str = match path.to_str() { + Some(d) => d, + None => { + e0031!(&path.to_string_lossy()); + continue; + } + }; + + let fsize = meta.len(); + + // if it is an empty file we don't need + // to check further + if fsize == 0 { + e0005!(dir_entry_str); + continue; + } + + let p = get_perms(&path); + if !owner_has(p, 4) { + e0002!(dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + fix_perms(&dir_entry_str); + } + continue; + } + let ftr = detective.analyze(&dir_entry_str); + //println!(">>> {:?}", ftr); + // we ignore errors from filetype recognition + if ftr.is_err() { + continue; + } + + let ft = ftr.unwrap(); + + // DEBUG !readme_symlinked.contains(dir_entry_str) + if ReadmeKind::No != *is_readme { + if !check_readme(dir_entry_str, is_readme, &ft) { + continue; + } + if ARGS.urlcheck { + lc.check_urls(dir_entry_str); + } + } + + match ft { + filemagic::Mimetype::Text | filemagic::Mimetype::TextCRLF => { + check_and_correct_perms(dir_entry_str, p); + let fext = get_extension_from_filename(&dir_entry_str); + if fext == Some("ins") || fext == Some("dtx") { + check_generated_files(&dir_entry_str, &mut generated); + } + match fext { + Some("bat") | Some("cmd") | + Some("nsh") | Some("reg") => (), + Some(_) | None => { + if ft == filemagic::Mimetype::TextCRLF { + e0012!(dir_entry_str); + if ARGS.correct_crlf { + fix_crlf(dir_entry_str); + } + } + } + }; + } + + filemagic::Mimetype::BOM(b) => { + //println!("{}: {} with BOM detected", dir_entry_str, b.as_ref()); + w0004!(dir_entry_str, b.as_ref()); + check_and_correct_perms(dir_entry_str, p); + } + filemagic::Mimetype::Binary + | filemagic::Mimetype::Script + | filemagic::Mimetype::ScriptCRLF => { + if !owner_has(p, 4) || !others_have(p, 4) { + e0002!(dir_entry_str, perms_to_string(p)); + }; + if ARGS.correct_perms { + fix_perms(&dir_entry_str); + } + } + filemagic::Mimetype::Pdf => { + check_and_correct_perms(dir_entry_str, p); + let ret = is_pdf_ok(&dir_entry_str); + if !ret.status { + e0017!(dir_entry_str); + if let Some(output) = ret.output { + println!("{}", &output); + }; + } + } + + filemagic::Mimetype::Archive | filemagic::Mimetype::Zip => { + w0001!(dir_entry_str); + check_and_correct_perms(dir_entry_str, p); + } + + filemagic::Mimetype::Data => check_and_correct_perms(dir_entry_str, p), + _ => continue, + } + if others_match(p, 0) { + e0002!(dir_entry_str, perms_to_string(p)); + if ARGS.correct_perms { + i0005!(&dir_entry_str); + set_perms(&dir_entry_str, 0o664); + } + } + if !(ARGS.ignore_dupes && tds_zip.is_none()) { + process(fsize, &path); + } + } + } + + print_casefolding(&lcnames); + print_generated(&doublenames, &generated); + print_doublenames(&doublenames); + + if ARGS.ignore_dupes && tds_zip.is_none() { + return None; + } + + // Set up thread pool for the task to hash a file. Number of CPUs + 1 has been + // found to be a good pool size, likely since the walker thread should be + // doing mostly IO. + let mut pool = Pool::new(num_cpus::get() as u32 + 1); + + let mut hashes: HashMap<(u64, Vec<u8>), DupPath> = HashMap::default(); + pool.scoped(|scope| { + let (tx, rx) = channel(); + + let hashref = &mut hashes; + scope.execute(move || { + for (size, path, hash) in rx.iter() { + hashref + .entry((size, hash)) + .or_insert_with(DupPath::new) + .push(path); + } + }); + + for size in sizes.keys() { + let paths = &sizes[size]; + if paths.len() == 1 && tds_zip.is_none() { + continue; + }; + + for p in &sizes[size] { + let txc = tx.clone(); + scope.execute(move || hash_file(*size, p.to_path_buf(), &txc)); + } + } + }); + + if !ARGS.ignore_dupes { + print_duplicates(&hashes); + } + Some(hashes) +} + +fn print_casefolding(hashes: &HashMap<PathBuf, Vec<(PathBuf, FileKind)>>) { + + for (k, eles) in hashes.iter() { + //println!("pcf: {:?}, {:?}", k, &eles); + if eles.len() == 1 { + continue; + } + + e0025!(k.display()); + + for (p, ty) in eles { + println!(" >>> {} ({})", p.display(), ty); + } + } +} + +fn print_generated(hashes: &HashMap<PathBuf, Vec<PathBuf>>, generated: &HashMap<PathBuf, PathBuf>) { + for (k, gen) in generated.iter() { + if hashes.contains_key(k) { + let v = &hashes[k]; + for fname in v { + e0019!(fname.to_str().unwrap(), gen.to_str().unwrap()); + } + } + } +} + +fn print_doublenames(hashes: &HashMap<PathBuf, Vec<PathBuf>>) { + for (k, paths) in hashes.iter() { + if paths.len() == 1 { + continue; + } + let ks = k.to_str().unwrap(); + if ks == "README" + || ks == "README.txt" + || ks == "README.md" + || ks == "Makefile" + || ks == "Makefile.am" + || ks == "Makefile.in" + || ks == "makefile" + { + continue; + } + + w0003!(k.to_str().unwrap()); + // println!(":: {}", k.display()); + + for p in paths { + println!(" >>> {}", p.display()); + } + } +} + +fn show_tmp_endings() { + i0006!(); + for (t, c) in temp_file_endings() { + println!("{:23} {}", t, c); + } +} diff --git a/support/pkgcheck/src/messages/errorsd.rs b/support/pkgcheck/src/messages/errorsd.rs new file mode 100644 index 0000000000..bd726754c0 --- /dev/null +++ b/support/pkgcheck/src/messages/errorsd.rs @@ -0,0 +1,445 @@ +// This file is generated by a Perl script. The source is +// in the docs/ directory of the repository. + +pub fn e0001d() { + println!( + r#" +E0001 -- Bad characters in file name + +File name should not contain non-ascii characters. Additionally, file +names should not contain control characters or other characters which +may have a special meaning for UNIX shells. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#nounixspecialcharacters +"# + ) +} + +pub fn e0002d() { + println!( + r#" +E0002 -- File Permissions + +Files submitted to CTAN should be world readable. + +Only files that are truly executable (like scripts and binaries) should +be marked as such. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#filepermissions +"# + ) +} + +pub fn e0003d() { + println!( + r#" +E0003 -- README is not a text file + +The README file specified in the error message must be a text file but +it isn't. +"# + ) +} + +pub fn e0004d() { + println!( + r#" +E0004 -- Empty directory not allowed + +Empty directories are considered as rubbish, and are usually not +accepted as part of a package. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noemptyfiles +"# + ) +} + +pub fn e0005d() { + println!( + r#" +E0005 -- Empty files not allowed + +Empty files are considered as rubbish, and are usually not accepted as +part of a package. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noemptyfiles +"# + ) +} + +pub fn e0006d() { + println!( + r#" +E0006 -- Hidden directories not allowed + +A package should not contain hidden directories. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles +"# + ) +} + +pub fn e0007d() { + println!( + r#" +E0007 -- Hidden files not allowed + +A package should not contain hidden files. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles +"# + ) +} + +pub fn e0008d() { + println!( + r#" +E0008 -- Temporary file detected + +A temporary file was detected. These are typically files created by TeX +& friends and should not be part of a package. + +Temporary files will also be detected in a TDS zip archive. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles +"# + ) +} + +pub fn e0009d() { + println!( + r#" +E0009 -- Package doesn't contain a README file + +A package must contain at least one of README, README.md or README.txt +file. + +For more details refer to: +http://mirrors.ibiblio.org/CTAN/help/ctan/CTAN-upload-addendum.html#readme +"# + ) +} + +pub fn e0010d() { + println!( + r#" +E0010 -- Broken symlink detected + +A broken symlink was detected. +"# + ) +} + +pub fn e0011d() { + println!( + r#" +E0011 -- Wrong permission for directory + +Directories should have rwx for the owner and at least r-x for others +(i.e. world readable). + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#filepermissions +"# + ) +} + +pub fn e0012d() { + println!( + r#" +E0012 -- CRLF line endings detected + +The file specified in the error message contains CRLF line endings. Text +files should have UNIX style line endings. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#crlf +"# + ) +} + +pub fn e0013d() { + println!( + r#" +E0013 -- Socket special fie detected + +The file specified in the error message is a socket special file which +is not allowed. +"# + ) +} + +pub fn e0014d() { + println!( + r#" +E0014 -- Fifo special file detected + +The file specified in the error message is a fifo special file which is +not allowed. +"# + ) +} + +pub fn e0015d() { + println!( + r#" +E0015 -- Bloch device file detected + +The file specified in the error message is a block device file which is +not allowed. +"# + ) +} + +pub fn e0016d() { + println!( + r#" +E0016 -- Character device file detected + +The file specified in the error message is a character device file which +is not allowed. +"# + ) +} + +pub fn e0017d() { + println!( + r#" +E0017 -- PDF document is in error + +The PDF document mentioned in the message is in error. + +pdfinfo will be run to check if a PDF document can be read. Message +E0017 will be followed by the error messages from pdfinfo. + +Example: + + I0002 Checking package files in directory somepkg + E0017 PDF error detected in somepkg/sompkg.pdf + Syntax Error (1293042): Illegal character ')' + Syntax Error: Couldn't find trailer dictionary + Syntax Error (1293042): Illegal character ')' + Syntax Error: Couldn't find trailer dictionary + Syntax Error: Couldn't read xref table +"# + ) +} + +pub fn e0018d() { + println!( + r#" +E0018 -- Unwanted directory detected + +A directory was detected which should not be part of a package. Example: +__MACOSX +"# + ) +} + +pub fn e0019d() { + println!( + r#" +E0019 -- Generated file detected + +In order to avoid redundancy we don't want to have included files in a +package which easily can be generated from other files in the +submission. + +Exceptions are the README files of the package, i.e. README, README.md +or README.txt. + +Starting with version 1.1.0 pkgcheck also detects generated files if +they are in a different directory in the package. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#nogeneratedfiles +"# + ) +} + +pub fn e0020d() { + println!( + r#" +E0020 -- Unwanted directory detected in the top level directory in TDS zip archive + +A top level directory of a TDS archive should only contain all or some +of the following directories: + +- tex +- fonts +- metafont +- metapost +- bibtex +- scripts +- doc +- source +"# + ) +} + +pub fn e0021d() { + println!( + r#" +E0021 -- Error when reading a file + +An error was encountered when reading the file specified in the message. +"# + ) +} + +pub fn e0022d() { + println!( + r#" +E0022 -- Check of an URL in a README file failed + +URL checking is in effect. An error occcured when trying to retrieve an +URL which was found in the specified README file. +"# + ) +} + +pub fn e0023d() { + println!( + r#" +E0023 -- Follow up error when trying to read a directory with insufficient permissions + +Error which is a follow-up error. For instance, when a directory could +not be read. +"# + ) +} + +pub fn e0024d() { + println!( + r#" +E0024 -- TDS zip archive has wrong permissions + +The TDS zip archive should have at least r-- for the owner and at least +r-- for others (i.e. world readable). + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#filepermissions +"# + ) +} + +pub fn e0025d() { + println!( + r#" +E0025 -- Duplicate names when ignoring letter case for files or directories + +As there are operating systems which do not distinguish between myfile +and MYFILE we don't want to have file names in a directory which are the +same after converting to lower case. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#filenames +"# + ) +} + +pub fn e0026d() { + println!( + r#" +E0026 -- Files not in TDS or different in TDS and non-install tree + +The file mentioned in the error message is either not existing in the +TDS zip archive, or it is different to the one in the non-install tree +"# + ) +} + +pub fn e0027d() { + println!( + r#" +E0027 -- An I/O error occured + +Some kind of I/O error occured. If you believe there is an error in +pkgcheck please contact the author. +"# + ) +} + +pub fn e0028d() { + println!( + r#" +E0028 -- A path name in a TDS zip archive must contain the package name + +The path names in a TDS zip archive must contain the package name. + +EXAMPLE: Assume a package somepkg. Then path names should look like +follows: + + tex/latex/somepkg/somepkg.cls + doc/latex/somepkg/README + source/latex/somepkg/somepkg.dtx + ... +"# + ) +} + +pub fn e0029d() { + println!( + r#" +E0029 -- README file: encoding with BOM detected + +A README file should be either ASCII or UTF-8 without BOM(byte order +mark) + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#readme +"# + ) +} + +pub fn e0030d() { + println!( + r#" +E0030 -- A symlink was found which points outside of the package directory tree + +A symlink must not point to a file or directory outside of the package +directory tree. +"# + ) +} + +pub fn e0031d() { + println!( + r#" +E0031 -- File name contains invalid UTF-8 character(s) + +A file name contains invald UTF-8 character(s). +"# + ) +} + +pub fn e0033d() { + println!( + r#" +E0033 -- Error when unpacking tds archive + +In order to investigate the contents of the TDS zip archive pkgcheck +unpacks the TDS zip archive to a temporary location which failed for the +reason given in the error message. +"# + ) +} + +pub fn e0034d() { + println!( + r#" +E0034 -- Unwanted file detected in the top level directory in TDS zip archive + +A top level directory of a TDS archive should only contain certain +directories but no files. +"# + ) +} + diff --git a/support/pkgcheck/src/messages/fatald.rs b/support/pkgcheck/src/messages/fatald.rs new file mode 100644 index 0000000000..b41cab83b1 --- /dev/null +++ b/support/pkgcheck/src/messages/fatald.rs @@ -0,0 +1,66 @@ +// This file is generated by a Perl script. The source is +// in the docs/ directory of the repository. + +pub fn f0001d() { + println!( + r#" +F0001 -- Specify a directory to check (use option -d) + +pkgcheck was called without any options. Use option -d to check a +directory +"# + ) +} + +pub fn f0002d() { + println!( + r#" +F0002 -- Specified directory does not exist. Exiting... + +The directory specified at the command line does not exit. +"# + ) +} + +pub fn f0003d() { + println!( + r#" +F0003 -- Specified TDS archive does not exist or is no file + +Specify a valid TDS zip archive when calling pkgcheck +"# + ) +} + +pub fn f0004d() { + println!( + r#" +F0004 -- The file specified as TDS archive is no zip archive + +Specify a valid TDS zip archive when calling pkgcheck +"# + ) +} + +pub fn f0005d() { + println!( + r#" +F0005 -- Bad file name for the zip archive + +pkgcheck detected that the file name of the TDS zip archive doesn't end +with .tds.zip +"# + ) +} + +pub fn f0006d() { + println!( + r#" +F0006 -- Unknown error code specified with option -e resp. ---explain. Exiting... + +pkgcheck was called with option -e resp. --explain, and an unknown error +code was specified. +"# + ) +} + diff --git a/support/pkgcheck/src/messages/informationd.rs b/support/pkgcheck/src/messages/informationd.rs new file mode 100644 index 0000000000..d4e3ea43b3 --- /dev/null +++ b/support/pkgcheck/src/messages/informationd.rs @@ -0,0 +1,77 @@ +// This file is generated by a Perl script. The source is +// in the docs/ directory of the repository. + +pub fn i0001d() { + println!( + r#" +I0001 -- Successfully converted from CRLF to LF + +Just an information that pkgcheck has successfully converted a file from +CRLF to LF line endings +"# + ) +} + +pub fn i0002d() { + println!( + r#" +I0002 -- Checking package files in directory + +Just an information that pkgcheck starts checking the package files in +the unzipped directory trees +"# + ) +} + +pub fn i0003d() { + println!( + r#" +I0003 -- Checking TDS zip archive + +Just an information that pkgcheck starts checking the TDS zip archive +"# + ) +} + +pub fn i0004d() { + println!( + r#" +I0004 -- Correcting line endings for file + +The file had CRLF line ending and will be corrected to have LF (Unix +like) line endings. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#crlf +"# + ) +} + +pub fn i0005d() { + println!( + r#" +I0005 -- Corrections permissions for file or directory + +pkgcheck corrects wrong permsissions for package files and directories. +It runs the chmod command in verbose mode. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#filepermissions +"# + ) +} + +pub fn i0006d() { + println!( + r#" +I0006 -- Files having one of the following file name endings are regarded as temporary + +Option --show-temp-endings was used, and pkgcheck prints a list of +temporary file endings and their meanings. + +For more details refer to: +http://mirror.ctan.org/help/ctan/CTAN-upload-addendum.html#noauxfiles +"# + ) +} + diff --git a/support/pkgcheck/src/messages/mod.rs b/support/pkgcheck/src/messages/mod.rs new file mode 100644 index 0000000000..c63b6e2cec --- /dev/null +++ b/support/pkgcheck/src/messages/mod.rs @@ -0,0 +1,657 @@ +mod errorsd; +mod fatald; +mod informationd; +mod warningsd; + +use self::errorsd::*; +use self::fatald::*; +use self::informationd::*; +use self::warningsd::*; + +macro_rules! no_colors { + () => { + $crate::ARGS.no_colors + }; +} + +macro_rules! error_occured { + () => { + $crate::ERROR_OCCURED.store(true, Ordering::Relaxed); + }; +} +macro_rules! yellow { + ($fmt:expr) => { + if no_colors!() { + $fmt.clear() + } else { + $fmt.bright_yellow() + } + }; +} + +// macro_rules! red { +// ($fmt:expr) => { +// if no_colors!() { +// $fmt.clear() +// } else { +// $fmt.bright_red() +// } +// }; +// } + +macro_rules! msgid { + ($fmt:expr) => { + match (no_colors!(), &$fmt[..1]) { + (true, _) => $fmt.clear(), + (false, "E") => $fmt.bright_red().bold(), + (false, "I") => $fmt.bright_yellow(), + (false, "W") => $fmt.bright_red(), + (false, "F") => $fmt.bright_red().bold(), + (_, _) => $fmt.clear(), + } + }; +} + +macro_rules! f0001 { + () => { + error_occured!(); + print!( + "{} Specify a directory to check (use option -d)\n", + msgid!("F0001") + ); + }; +} + +macro_rules! f0002 { + ($fmt1:expr) => { + error_occured!(); + print!( + "{} Specified directory {} does not exist. Exiting...\n", + msgid!("F0002"), + $fmt1 + ); + }; +} + +macro_rules! f0003 { + ($fmt1:expr) => { + error_occured!(); + print!( + "{} Specified TDS archive {} does not exist or is no file. Exiting...\n", + msgid!("F0003"), + $fmt1 + ); + }; +} + +macro_rules! f0004 { + ($fmt1:expr) => { + error_occured!(); + print!( + "{} File {} specified as TDS archive is no zip archive. Exiting...\n", + msgid!("F0004"), + $fmt1 + ); + }; +} + +macro_rules! f0005 { + ($fmt1:expr) => { + error_occured!(); + print!( + "{} Bad filename {} for the tds.zip archive. Exiting...\n", + msgid!("F0005"), + $fmt1 + ); + }; +} + +// macro_rules! f0006 { +// ($fmt1:expr) => { +// error_occured!(); +// print!( +// "{} Unknown error code `{}` specified with option -e resp. ---explain. Exiting...\n", +// msgid!("F0006"), +// $fmt1 +// ); +// }; +// } + +macro_rules! e0001 { + ($fmt1:expr, $fmt2:expr, $fmt3:expr) => { + error_occured!(); + print!( + "{} Bad character {} ({:#02x}) found in file name {} at offset {}\n", + msgid!("E0001"), + $fmt1, + $fmt1 as usize, + $fmt2, + $fmt3 + ); + }; +} + +macro_rules! e0002 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} File {} has bad permissions: {}\n", + msgid!("E0002"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0003 { + ($fmt:expr) => { + error_occured!(); + print!("{} {} is no text file\n", msgid!("E0003"), $fmt); + }; +} + +macro_rules! e0004 { + ($fmt:expr) => { + error_occured!(); + print!("{} Empty directory {} detected\n", msgid!("E0004"), $fmt); + }; +} + +macro_rules! e0005 { + ($fmt:expr) => { + error_occured!(); + print!("{} Empty file {} detected\n", msgid!("E0005"), $fmt); + }; +} + +macro_rules! e0006 { + ($fmt:expr) => { + error_occured!(); + print!("{} Hidden directory {} detected\n", msgid!("E0006"), $fmt); + }; +} + +macro_rules! e0007 { + ($fmt:expr) => { + error_occured!(); + print!("{} Hidden file {} detected\n", msgid!("E0007"), $fmt); + }; +} + +macro_rules! e0008 { + ($fmt:expr) => { + error_occured!(); + print!("{} Temporary file {} detected\n", msgid!("E0008"), $fmt); + }; +} + +macro_rules! e0009 { + () => { + error_occured!(); + print!( + "{} One of README/README.md/README.txt must exist\n", + msgid!("E0009") + ); + }; +} + +macro_rules! e0010 { + ($fmt:expr) => { + error_occured!(); + print!("{} {} is a broken symlink\n", msgid!("E0010"), $fmt); + }; +} + +macro_rules! e0011 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} Directory {} has bad permissions: {}\n", + msgid!("E0011"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0012 { + ($fmt:expr) => { + error_occured!(); + print!("{} CRLF detected: {}\n", msgid!("E0012"), $fmt); + }; +} + +macro_rules! e0013 { + ($fmt:expr) => { + error_occured!(); + print!("{} {} is a socket special file\n", msgid!("E0013"), $fmt); + }; +} + +macro_rules! e0014 { + ($fmt:expr) => { + error_occured!(); + print!("{} {} is a fifo file\n", msgid!("E0014"), $fmt); + }; +} + +macro_rules! e0015 { + ($fmt:expr) => { + error_occured!(); + print!("{} {} is a block device file\n", msgid!("E0015"), $fmt); + }; +} + +macro_rules! e0016 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} {} is a character device file\n", + msgid!("E0016"), + $fmt + ); + }; +} + +macro_rules! e0017 { + ($fmt:expr) => { + error_occured!(); + print!("{} PDF error detected in {}\n", msgid!("E0017"), $fmt); + }; +} + +macro_rules! e0018 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Unwanted directory {} detected\n", + msgid!("E0018"), + $fmt + ); + }; +} + +macro_rules! e0019 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} {} generated by {} exists\n", + msgid!("E0019"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0020 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Unwanted directory `{}` detected in the top level directory of a TDS archive\n", + msgid!("E0020"), + $fmt + ); + }; +} + +macro_rules! e0021 { + ($fmt1:expr, $fmt2:expr, $fmt3:expr) => { + error_occured!(); + print!( + "{} File {} : Error reading in line {}: {}\n", + msgid!("E0021"), + $fmt1, + $fmt2, + $fmt3 + ); + }; +} + +macro_rules! e0022 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!("{} {}: `{}`\n", msgid!("E0022"), $fmt1, $fmt2); + }; +} + +macro_rules! e0023 { + ($fmt:expr) => { + error_occured!(); + print!("{} {}\n", msgid!("E0023"), $fmt); + }; +} + +macro_rules! e0024 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} The TDS zip archive {} has bad permissions: {}\n", + msgid!("E0024"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0025 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Duplicate names when ignoring letter case for: {}\n", + msgid!("E0025"), + $fmt + ); + }; +} + +macro_rules! e0026 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} {} : file not in TDS or different in TDS and non-install tree\n", + msgid!("E0026"), + $fmt + ); + }; +} + +macro_rules! e0027 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} {}: An I/O error occured -> {}\n", + msgid!("E0027"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0028 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} No directory {} (= package name) found in path {}\n", + msgid!("E0028"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0029 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} {}: {} encoding with BOM detected\n", + msgid!("E0029"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0030 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} Symlink {} points to {} which is outside of the package directory tree\n", + msgid!("E0030"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! e0031 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} File name {} contains invalid UTF-8 character(s)\n", + msgid!("E0031"), + $fmt + ); + }; +} + +macro_rules! e0033 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} Error `{}` when unpacking tds archive {}. Exiting...\n", + msgid!("E0033"), + $fmt2, + $fmt1 + ); + }; +} + +macro_rules! e0034 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Unwanted file `{}` detected in the top level directory of a TDS archive\n", + msgid!("E0034"), + $fmt + ); + }; +} + +macro_rules! w0001 { + ($fmt:expr) => { + print!( + "{} Archive as package file detected: {}\n", + msgid!("W0001"), + $fmt + ); + }; +} + +macro_rules! w0002 { + () => { + print!("{} Duplicate files detected\n", msgid!("W0002"),); + }; +} + +macro_rules! w0003 { + ($fmt:expr) => { + error_occured!(); + print!( + "{} Same named files detected in the package directory tree: {}\n", + msgid!("W0003"), + $fmt + ); + }; +} + +macro_rules! w0004 { + ($fmt1:expr, $fmt2:expr) => { + error_occured!(); + print!( + "{} {}: {} encoding with BOM detected\n", + msgid!("W0004"), + $fmt1, + $fmt2 + ); + }; +} + +macro_rules! i0002 { + ($fmt:expr) => { + print!( + "{} {} {}\n", + msgid!("I0002"), + yellow!("Checking package files in directory"), + yellow!($fmt) + ); + }; +} + +macro_rules! i0003 { + ($fmt:expr) => { + print!( + "{} {} {}\n", + msgid!("I0003"), + yellow!("Checking TDS zip archive"), + yellow!($fmt) + ); + }; +} + +macro_rules! i0004 { + ($fmt:expr) => { + print!( + "{} Correcting line endings for file {}\n", + msgid!("I0004"), + $fmt + ); + }; +} + +macro_rules! i0005 { + ($fmt:expr) => { + print!( + "{} Correcting permissions for {}\n", + msgid!("I0005"), + $fmt + ); + }; +} + +macro_rules! i0006 { + () => { + print!( + "{} Files having one of the following file name endings are regarded as temporary\n", + msgid!("I0006") + ); + }; +} + +macro_rules! i0007 { + ($fmt:expr) => { + print!( + "{} {}: Successfully converted from CRLF to LF\n", + msgid!("I0007"), + $fmt + ); + }; +} + +pub fn explains(err: &str) { + let err_upp = err.to_ascii_uppercase(); + match err_upp.as_str() { + "F0001" => f0001d(), + "F0002" => f0002d(), + "F0003" => f0003d(), + "F0004" => f0004d(), + "F0005" => f0005d(), + "F0006" => f0006d(), + + "E0001" => e0001d(), + "E0002" => e0002d(), + "E0003" => e0003d(), + "E0004" => e0004d(), + "E0005" => e0005d(), + "E0006" => e0006d(), + "E0007" => e0007d(), + "E0008" => e0008d(), + "E0009" => e0009d(), + "E0010" => e0010d(), + "E0011" => e0011d(), + "E0012" => e0012d(), + "E0013" => e0013d(), + "E0014" => e0014d(), + "E0015" => e0015d(), + "E0016" => e0016d(), + "E0017" => e0017d(), + "E0018" => e0018d(), + "E0019" => e0019d(), + "E0020" => e0020d(), + "E0021" => e0021d(), + "E0022" => e0022d(), + "E0023" => e0023d(), + "E0024" => e0024d(), + "E0025" => e0025d(), + "E0026" => e0026d(), + "E0027" => e0027d(), + "E0028" => e0028d(), + "E0029" => e0029d(), + "E0030" => e0030d(), + "E0031" => e0031d(), + "E0033" => e0033d(), + "E0034" => e0034d(), + + // "I0001" => i0001d!(), + "I0001" => i0001d(), + "I0002" => i0002d(), + "I0003" => i0003d(), + "I0004" => i0004d(), + "I0005" => i0005d(), + "I0006" => i0006d(), + + "W0001" => w0001d(), + "W0002" => w0002d(), + "W0003" => w0003d(), + "W0004" => w0004d(), + + e => println!( + "F0006 Unknown error code `{}` specified with option -e resp. --explain. Exiting...", + e + ), + } +} + +pub fn explains_all() { + explains("F0001"); + explains("F0002"); + explains("F0003"); + explains("F0004"); + explains("F0005"); + explains("F0006"); + + explains("E0001"); + explains("E0002"); + explains("E0003"); + explains("E0004"); + explains("E0005"); + explains("E0006"); + explains("E0007"); + explains("E0008"); + explains("E0009"); + explains("E0010"); + explains("E0011"); + explains("E0012"); + explains("E0013"); + explains("E0014"); + explains("E0015"); + explains("E0016"); + explains("E0017"); + explains("E0018"); + explains("E0019"); + explains("E0020"); + explains("E0021"); + explains("E0022"); + explains("E0023"); + explains("E0024"); + explains("E0025"); + explains("E0026"); + explains("E0027"); + explains("E0028"); + explains("E0029"); + explains("E0030"); + explains("E0031"); + explains("E0033"); + explains("E0034"); + + explains("I0001"); + explains("I0002"); + explains("I0003"); + explains("I0004"); + explains("I0005"); + explains("I0006"); + + explains("W0001"); + explains("W0002"); + explains("W0003"); + explains("W0004"); +} diff --git a/support/pkgcheck/src/messages/warningsd.rs b/support/pkgcheck/src/messages/warningsd.rs new file mode 100644 index 0000000000..588144968d --- /dev/null +++ b/support/pkgcheck/src/messages/warningsd.rs @@ -0,0 +1,58 @@ +// This file is generated by a Perl script. The source is +// in the docs/ directory of the repository. + +pub fn w0001d() { + println!( + r#" +W0001 -- Archive as package file detected + +Usually a CTAN package should not contain archives. An exception are +situations where, for example, the source code of a package is kept in a +separate zip archive. +"# + ) +} + +pub fn w0002d() { + println!( + r#" +W0002 -- Duplicate files detected + +Duplicate files were detected which are listed right after this message. + +The message is a warning message as something like this could not be +seen as an error in general. +"# + ) +} + +pub fn w0003d() { + println!( + r#" +W0003 -- Same named files detected in the package tree + +We like to have unique file names over the whole package directory tree. +When we discover same named files we report it as a warning. Common +names like README, README.txt, README.md, Makefile, Makefile.in, +Makefile.am and makefile are ignored when checking. + +For more details refer to: +http://mirror.utexas.edu/ctan/help/ctan/CTAN-upload-addendum.html#uniquefilenames +"# + ) +} + +pub fn w0004d() { + println!( + r#" +W0004 -- encoding with BOM detected + +A UTF encoded package file contains a BOM (byte order mark). Currently, +we issues a warning. + +Nevertheless, the CTAN team discourages uses of BOM. Please be aware, +that in some future time this could be reagarded as an error. +"# + ) +} + diff --git a/support/pkgcheck/src/recode.rs b/support/pkgcheck/src/recode.rs new file mode 100644 index 0000000000..39a85bacd2 --- /dev/null +++ b/support/pkgcheck/src/recode.rs @@ -0,0 +1,62 @@ +// the code here is taken (and slightly modified) from https://github.com/XadillaX/crlf2lf + +// +// convert CRLF (0D 0A) to LF ( 0A ) +// + +use std::fs::File; +use std::io::prelude::*; +use std::io::{self, Read}; +use std::path::Path; + +// The caller has to make sure that crlf2lf is only +// used for text files +pub fn crlf2lf(fname: &str) -> Result<(), io::Error> { + let path = Path::new(fname); + + let mut hdl_in; + match File::open(path) { + Ok(f) => { + hdl_in = f; + } + Err(e) => return Err(e), + }; + let mut buffer: Vec<u8> = Vec::new(); + + // read the whole file + if let Err(e) = hdl_in.read_to_end(&mut buffer) { + return Err(e); + }; + + // convert \r\n to \n + let mut another_vec: Vec<u8> = Vec::new(); + const CR: u8 = 0x0d; // 13 + const LF: u8 = 0x0a; // 10 + + for i in 0..buffer.len() { + if buffer[i] == CR { + if i < buffer.len() - 1 && buffer[i + 1] == LF { + continue; + } + + if i > 0 && buffer[i - 1] == LF { + continue; + } + } + another_vec.push(buffer[i]); + } + + let mut hdl_out; + match File::create(path) { + Ok(f) => { + hdl_out = f; + } + Err(e) => return Err(e), + }; + + // write back + match hdl_out.write(&another_vec) { + Ok(_) => Ok(()), + Err(e) => Err(e), + } +} diff --git a/support/pkgcheck/src/utils.rs b/support/pkgcheck/src/utils.rs new file mode 100644 index 0000000000..6b2d0a8753 --- /dev/null +++ b/support/pkgcheck/src/utils.rs @@ -0,0 +1,396 @@ +use std::fs::read_dir; +use std::fs::read_link; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::path::PathBuf; +use walkdir::DirEntry; + +use regex::Regex; +use std::borrow::Cow; +use std::io; +use std::process::Command; + + +use std::fs; + +pub fn temp_file_endings() -> Vec<(&'static str, &'static str)> { + // https://github.com/github/gitignore/blob/master/TeX.gitignore + // http://hopf.math.purdue.edu/doc/html/suffixes.html + vec![ + ("-blx.aux", "bibliography auxiliary file"), + ("-blx.bib", "bibliography auxiliary file"), + (".4ct", "htlatex related"), + (".4tc", "htlatex related"), + (".DS_Store", "Mac OS custom attribute file"), + (".acn", "glossaries related"), + (".acr", "glossaries related"), + (".alg", ""), + (".aux", "core latex auxiliary file"), + (".backup", "backup file"), + (".bak", "backup file"), + (".bbl", "bibliography auxiliary file"), + (".bcf", "bibliography auxiliary file"), + (".blg", "bibliography log file"), + (".brf", "hyperref related"), + (".cb", "core latex auxiliary file"), + (".cb2", "core latex auxiliary file"), + (".cpt", "cprotect related"), + (".dvi", "intermediate document"), + (".ent", ""), + (".fdb_latexmk", "latexmk related"), + (".fff", "endfloat related"), + (".fls", ""), + (".fmt", "core latex auxiliary file"), + (".fot", "core latex auxiliary file"), + (".gaux", "generated by gregoriotex"), + (".glg", "glossary related"), + (".glo", "glossary related"), + (".glog", "generated by gregoriotex"), + (".gls", "glossary related"), + (".glsdefs", "glossaries related"), + (".gtex", "generated by gregoriotex"), + (".idv", "htlatex related"), + (".idx", "makeidx related"), + (".ilg", "makeidx related"), + (".ind", "makeidx related"), + (".lg", "htlatex related"), + (".loa", "core latex auxiliary file (list of algorithms)"), + (".lod", "generated by easy-todo"), + (".lof", "core latex auxiliary file (list of figures)"), + (".log", "a log file for any flavor of TeX"), + (".lol", "core latex auxiliary file (list of listings)"), + (".los", "list of slides"), + (".lot", "core latex auxiliary file"), + (".lox", ""), + (".lyx#", "LyX related autosave file"), + (".maf", "generated by minitoc"), + (".mlc", "generated by minitoc"), + (".mlf", "generated by minitoc"), + (".mlt", "generated by minitoc"), + (".nav", "beamer related"), + (".nlg", ""), + (".nlo", ""), + (".nls", ""), + (".o", "C object file"), + (".out", "Core latex auxiliary file"), + (".pdfsync", "pdfsync related"), + (".pre", "beamer related"), + (".pyg", ""), + (".run.xml", ""), + (".sav", "used for saved data"), + (".snm", "beamer related"), + (".soc", ""), + (".spl", "elsarticle related"), + (".sta", "generated by standalone package"), + (".swp", "vim swap file"), + (".synctex", "synctex related"), + (".synctex(busy)", "synctex related"), + (".synctex.gz", "synctex related"), + (".synctex.gz(busy)", "synctex related"), + (".tdo", "generated by todonotes (list of todos)"), + (".thm", "amsthm related"), + (".tmb", "generated by thumbs package"), + (".tmp", "indicates a temporary file"), + (".toc", "core latex auxiliary file (table of contents)"), + (".trc", "htlatex related"), + (".ttt", "endfloat related"), + (".tuc", ""), + (".upa", "generated by the soulpos package"), + (".upb", "generated by the soulpos package"), + (".url", "generated by jurabib"), + (".vrb", "beamer related"), + (".w18", "temporary file for the ifplatform package"), + (".xdv", "intermediate document"), + (".xref", "htlatex related"), + ("~", "a file name ending with ~ (tilde) is temporary anyway"), + // ( ".lyx~", "LyX related backup file" ), + ] +} + +pub fn regex_temporary_file_endings() -> Regex { + let mut rv = String::new(); + let mut first_time = true; + for (p, _) in temp_file_endings() { + if first_time { + rv.push('('); + first_time = false; + } else { + rv.push('|'); + } + let px = str::replace(p, ".", "\\."); + + rv.push_str(&px); + } + rv.push_str(")$"); + Regex::new(&rv).unwrap() +} + +pub fn is_temporary_file(entry: &str) -> bool { + lazy_static! { + static ref RE: Regex = regex_temporary_file_endings(); + } + + RE.is_match(entry) +} + +pub fn is_empty_directory(entry: &DirEntry) -> Result<bool, io::Error> { + let s = entry.path().to_str().unwrap(); + match read_dir(s) { + // try to read the directory specified + Ok(contents) => Ok(contents.count() == 0), + Err(e) => Err(e), + } +} + +pub fn is_unwanted_directory(entry: &str) -> bool { + let names = ["__MACOSX"]; + for n in &names { + if *n == entry { + return true; + }; + } + + false +} + +// is_hidden checks for files or directories starting with a dot +pub fn is_hidden(entry: &str) -> bool { + entry.starts_with('.') +} + +// is_hidden checks for files or directories starting with a dot +pub fn is_hidden_directory(entry: &DirEntry) -> bool { + let fname = entry.file_name(); + + if fname == "." { + return false; + }; + if fname == "./" { + return false; + }; + + fname.to_str().map(|s| s.starts_with('.')).unwrap_or(false) +} + +pub fn get_symlink(entry: &DirEntry) -> Result<Option<PathBuf>, io::Error> { + let r = entry.path().to_str().unwrap(); + + match read_link(r) { + Ok(o) => { + let full_path = if o.is_absolute() { + o + } else { + // make the relative path absolute + let p = entry.path().parent().unwrap(); + p.join(&o) + }; + if full_path.exists() { + Ok(Some(full_path)) + } else { + Ok(None) + } + } + Err(e) => Err(e), + } +} + +pub fn _is_symlink_broken(entry: &DirEntry) -> Result<bool, io::Error> { + let r = entry.path().to_str().unwrap(); + + match read_link(r) { + Ok(o) => { + let full_path = if o.is_absolute() { + o + } else { + // make the relative path absolute + let p = entry.path().parent().unwrap(); + p.join(&o) + }; + Ok(!full_path.exists()) + } + Err(e) => Err(e), + } +} + +// Runs `pdfinfo` to check a PDF document. If `pdfinfo` +// returns non zero we assume that the PDF document is +// corrupted. +pub fn is_pdf_ok(fname: &str) -> CmdReturn { + run_cmd("/usr/bin/pdfinfo", &[fname]) +} + +pub fn get_perms(path: &Path) -> u32 { + match path.metadata() { + Ok(p) => p.permissions().mode(), + Err(_e) => 0, + } +} + +pub fn others_match(p: u32, m: u32) -> bool { + p == m +} + +#[test] +fn test_others_have() { + assert_eq!(others_have(0o600, 4), false); + assert_eq!(others_have(0o601, 4), false); + assert_eq!(others_have(0o602, 4), false); + assert_eq!(others_have(0o603, 4), false); + assert_eq!(others_have(0o604, 4), true); + assert_eq!(others_have(0o605, 4), true); + assert_eq!(others_have(0o606, 4), true); + assert_eq!(others_have(0o607, 4), true); +} + +// It checks if a permission `p` has the bits +// given in `m` set for others +pub fn others_have(p: u32, m: u32) -> bool { + let p1 = p & 0o0007; + p1 & m == m +} + +#[test] +fn test_owner_has() { + assert_eq!(owner_has(0o000, 4), false); + assert_eq!(owner_has(0o100, 4), false); + assert_eq!(owner_has(0o200, 4), false); + assert_eq!(owner_has(0o300, 4), false); + assert_eq!(owner_has(0o400, 4), true); + assert_eq!(owner_has(0o505, 4), true); + assert_eq!(owner_has(0o605, 4), true); + assert_eq!(owner_has(0o705, 4), true); +} + +// It checks if a permission `p` has the bits +// given in `m` set for the owner. +pub fn owner_has(p: u32, m: u32) -> bool { + let p1 = p & 0o7777; + let m1 = m << 6; + p1 & m1 == m1 +} + +#[allow(dead_code)] +fn owner_match(p: u32, m: u32) -> bool { + let p1 = p & 0o0700; + + let m1 = m << 6; + p1 == m1 +} + +// Formats a permission value to octal for output +pub fn perms_to_string(p: u32) -> String { + format!("{:04o}", p & 0o7777) +} + +pub struct CmdReturn { + pub status: bool, + pub output: Option<String>, +} + +// Runs a command in a shell +// If the command returns 0 stdout gets captured. Otherwise +// stderr gets captured and returned. +pub fn run_cmd(cmd: &str, argument: &[&str]) -> CmdReturn { + let output = Command::new(cmd) + .args(argument.iter()) + .output() + .expect("Failed to execute process"); + + if output.status.success() { + CmdReturn { + status: true, + output: Some(String::from_utf8_lossy(&output.stdout).to_string()), + } + } else { + CmdReturn { + status: false, + output: Some(String::from_utf8_lossy(&output.stderr).to_string()), + } + } +} + +// Sets permissions for a file or directory +// Sample invocation: set_perms("somfile", 0o644); +pub fn set_perms(entry: &str, p: u32) -> bool { + let ps = &format!("{:o}", p); + let rc = run_cmd("chmod", &["-v", ps, entry]); + + if rc.status { + if let Some(op) = rc.output { + print!("{}", op); + } + true + } else { + false + } +} + +// returns true if file is a directory and does exist +// returns false otherwise +pub fn exists_dir(file: &str) -> bool { + match fs::metadata(file) { + Ok(attr) => attr.is_dir(), + Err(_) => false, + } +} + +// returns true if file is a regular file and does exist +// returns false otherwise +pub fn exists_file(file: &str) -> bool { + match fs::metadata(file) { + Ok(attr) => attr.is_file(), + Err(_) => false, + } +} + +#[test] +fn test_dirname() { + assert!(dirname("/etc/fstab") == Some("/etc")); + assert!(dirname("/etc/") == Some("/etc")); + assert!(dirname("/") == Some("/")); +} + +#[allow(dead_code)] +pub fn dirname(entry: &str) -> Option<&str> { + if entry.ends_with('/') { + if entry.len() == 1 { + return Some(entry); + } + return Some(&entry[..entry.len() - 1]); + } + let pos = entry.rfind('/'); + match pos { + None => None, + Some(pos) => Some(&entry[..pos]), + } +} + +#[test] +fn test_filename() { + assert!(filename("/etc/fstab") == Some("fstab")); + assert!(filename("fstab") == Some("fstab")); + assert!(filename("../pkgcheck.rs/testdirs/fira.tds.zip") == Some("fira.tds.zip")); + assert!(filename("/etc/") == None); + assert!(filename("/") == None); +} + +pub fn filename(entry: &str) -> Option<&str> { + if entry.ends_with('/') { + return None; + } + + let pos = entry.rfind('/'); + match pos { + None => Some(entry), + Some(pos) => Some(&entry[pos + 1..]), + } +} + +// Found here: https://codereview.stackexchange.com/questions/98536/extracting-the-last-component-basename-of-a-filesystem-path +pub fn basename(path: &str) -> Cow<str> { + let mut pieces = path.rsplitn(2, |c| c == '/' || c == '\\'); + match pieces.next() { + Some(p) => p.into(), + None => path.into(), + } +} |