001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017
018package org.apache.commons.net.ftp.parser;
019
020import java.text.ParseException;
021import java.util.List;
022
023import org.apache.commons.net.ftp.FTPClientConfig;
024import org.apache.commons.net.ftp.FTPFile;
025
026/**
027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS
028 * Systems.
029 *
030 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for
031 *      usage instructions)
032 */
033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl {
034
035    static final int UNKNOWN_LIST_TYPE = -1;
036    static final int FILE_LIST_TYPE = 0;
037    static final int MEMBER_LIST_TYPE = 1;
038    static final int UNIX_LIST_TYPE = 2;
039    static final int JES_LEVEL_1_LIST_TYPE = 3;
040    static final int JES_LEVEL_2_LIST_TYPE = 4;
041
042    private int isType = UNKNOWN_LIST_TYPE;
043
044    /**
045     * Fallback parser for Unix-style listings
046     */
047    private UnixFTPEntryParser unixFTPEntryParser;
048
049    /**
050     * Dates are ignored for file lists, but are used for member lists where
051     * possible
052     */
053    static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18
054                                                                    // 13:52
055
056    /**
057     * Matches these entries:
058     *
059     * <pre>
060     *  Volume Unit    Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname
061     *  B10142 3390   2006/03/20  2   31  F       80    80  PS   MDI.OKL.WORK
062     * </pre>
063     *
064     * @see <a href=
065     *      "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data
066     *      set record formats</a>
067     */
068    static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume
069                                                                // ignored
070            "\\S+\\s+" + // unit - ignored
071            "\\S+\\s+" + // access date - ignored
072            "\\S+\\s+" + // extents -ignored
073            // If the values are too large, the fields may be merged (NET-639)
074            "(?:\\S+\\s+)?" + // used - ignored
075            "(?:F|FB|V|VB|U)\\s+" + // recfm - F[B], V[B], U
076            "\\S+\\s+" + // logical record length -ignored
077            "\\S+\\s+" + // block size - ignored
078            "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist
079            // but only support: PS, PO, PO-E
080            "(\\S+)\\s*"; // Dataset Name (file name)
081
082    /**
083     * Matches these entries:
084     * <pre>
085     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
086     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
087     * </pre>
088     */
089    static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name
090            "\\S+\\s+" + // version, modification (ignored)
091            "\\S+\\s+" + // create date (ignored)
092            "(\\S+)\\s+" + // modification date
093            "(\\S+)\\s+" + // modification time
094            "\\S+\\s+" + // size in lines (ignored)
095            "\\S+\\s+" + // size in lines at creation(ignored)
096            "\\S+\\s+" + // lines modified (ignored)
097            "\\S+\\s*"; // id of user who modified (ignored)
098
099    /**
100     * Matches these entries, note: no header:
101     * <pre>
102     *   IBMUSER1  JOB01906  OUTPUT    3 Spool Files
103     *   012345678901234567890123456789012345678901234
104     *             1         2         3         4
105     * </pre>
106     */
107    static final String JES_LEVEL_1_LIST_REGEX =
108            "(\\S+)\\s+" + // job name ignored
109            "(\\S+)\\s+" + // job number
110            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE)
111            "(\\S+)\\s+" + // number of spool files
112            "(\\S+)\\s+" + // Text "Spool" ignored
113            "(\\S+)\\s*" // Text "Files" ignored
114    ;
115
116    /**
117     * JES INTERFACE LEVEL 2 parser
118     * Matches these entries:
119     * <pre>
120     * JOBNAME  JOBID    OWNER    STATUS CLASS
121     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
122     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
123     * </pre>
124     * Sample output from FTP session:
125     * <pre>
126     * ftp> quote site filetype=jes
127     * 200 SITE command was accepted
128     * ftp> ls
129     * 200 Port request OK.
130     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
131     * JOBNAME  JOBID    OWNER    STATUS CLASS
132     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000 3 spool files
133     * IBMUSER  TSU01830 IBMUSER  OUTPUT TSU      ABEND=522 3 spool files
134     * 250 List completed successfully.
135     * ftp> ls job01906
136     * 200 Port request OK.
137     * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER
138     * JOBNAME  JOBID    OWNER    STATUS CLASS
139     * IBMUSER1 JOB01906 IBMUSER  OUTPUT A        RC=0000
140     * --------
141     * ID  STEPNAME PROCSTEP C DDNAME   BYTE-COUNT
142     * 001 JES2              A JESMSGLG       858
143     * 002 JES2              A JESJCL         128
144     * 003 JES2              A JESYSMSG       443
145     * 3 spool files
146     * 250 List completed successfully.
147     * </pre>
148     */
149
150    static final String JES_LEVEL_2_LIST_REGEX =
151            "(\\S+)\\s+" + // job name ignored
152            "(\\S+)\\s+" + // job number
153            "(\\S+)\\s+" + // owner ignored
154            "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored
155            "(\\S+)\\s+" + // job class ignored
156            "(\\S+).*" // rest ignored
157    ;
158
159    /*
160     * ---------------------------------------------------------------------
161     * Very brief and incomplete description of the zOS/MVS-file system. (Note:
162     * "zOS" is the operating system on the mainframe, and is the new name for
163     * MVS)
164     *
165     * The file system on the mainframe does not have hierarchal structure as for
166     * example the unix file system. For a more comprehensive description, please
167     * refer to the IBM manuals
168     *
169     * @LINK:
170     * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS
171     *
172     *
173     * Dataset names =============
174     *
175     * A dataset name consist of a number of qualifiers separated by '.', each
176     * qualifier can be at most 8 characters, and the total length of a dataset
177     * can be max 44 characters including the dots.
178     *
179     *
180     * Dataset organisation ====================
181     *
182     * A dataset represents a piece of storage allocated on one or more disks.
183     * The structure of the storage is described with the field dataset
184     * organinsation (DSORG). There are a number of dataset organisations, but
185     * only two are usable for FTP transfer.
186     *
187     * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E:
188     * extended partitioned dataset
189     *
190     * The PS file is just a flat file, as you would find it on the unix file
191     * system.
192     *
193     * The PO and PO-E files, can be compared to a single level directory
194     * structure. A PO file consist of a number of dataset members, or files if
195     * you will. It is possible to CD into the file, and to retrieve the
196     * individual members.
197     *
198     *
199     * Dataset record format =====================
200     *
201     * The physical layout of the dataset is described on the dataset itself.
202     * There are a number of record formats (RECFM), but just a few is relavant
203     * for the FTP transfer.
204     *
205     * Any one beginning with either F or V can safely used by FTP transfer. All
206     * others should only be used with great care.
207     * F means a fixed number of records per
208     * allocated storage, and V means a variable number of records.
209     *
210     *
211     * Other notes ===========
212     *
213     * The file system supports automatically backup and retrieval of datasets.
214     * If a file is backed up, the ftp LIST command will return: ARCIVE Not
215     * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST
216     *
217     *
218     * Implementation notes ====================
219     *
220     * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning
221     * with F or V or U, is fully parsed.
222     *
223     * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set.
224     * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name
225     * FTPFile.Timestamp: change time or null
226     *
227     *
228     *
229     * Additional information ======================
230     *
231     * The MVS ftp server supports a number of features via the FTP interface.
232     * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2>
233     * SEQ is the default and used for normal file transfer JES is used to
234     * interact with the Job Entry Subsystem (JES) similar to a job scheduler
235     * DB2 is used to interact with a DB2 subsystem
236     *
237     * This parser supports SEQ and JES.
238     *
239     *
240     *
241     *
242     *
243     *
244     */
245
246    /**
247     * The sole constructor for a MVSFTPEntryParser object.
248     *
249     */
250    public MVSFTPEntryParser() {
251        super(""); // note the regex is set in preParse.
252        super.configure(null); // configure parser with default configurations
253    }
254
255    /**
256     * Parses a line of an z/OS - MVS FTP server file listing and converts it
257     * into a usable format in the form of an <code> FTPFile </code> instance.
258     * If the file listing line doesn't describe a file, then
259     * <code> null </code> is returned. Otherwise a <code> FTPFile </code>
260     * instance representing the file is returned.
261     *
262     * @param entry
263     *            A line of text from the file listing
264     * @return An FTPFile instance corresponding to the supplied entry
265     */
266    @Override
267    public FTPFile parseFTPEntry(final String entry) {
268        if (isType == FILE_LIST_TYPE) {
269            return parseFileList(entry);
270        } else if (isType == MEMBER_LIST_TYPE) {
271            return parseMemberList(entry);
272        } else if (isType == UNIX_LIST_TYPE) {
273             return unixFTPEntryParser.parseFTPEntry(entry);
274        } else if (isType == JES_LEVEL_1_LIST_TYPE) {
275            return parseJeslevel1List(entry);
276        } else if (isType == JES_LEVEL_2_LIST_TYPE) {
277            return parseJeslevel2List(entry);
278        }
279
280        return null;
281    }
282
283    /**
284     * Parse entries representing a dataset list. Only datasets with DSORG PS or
285     * PO or PO-E and with RECFM F[B], V[B], U will be parsed.
286     *
287     * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred
288     * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80
289     * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device
290     * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO
291     * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB
292     *
293     * ----------------------------------- Group within Regex [1] Volume [2]
294     * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record
295     * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg:
296     * Dataset organisation. Many exists but only support: PS, PO, PO-E [10]
297     * Dsname: Dataset name
298     *
299     * Note: When volume is ARCIVE, it means the dataset is stored somewhere in
300     * a tape archive. These entries is currently not supported by this parser.
301     * A null value is returned.
302     *
303     * @param entry zosDirectoryEntry
304     * @return null: entry was not parsed.
305     */
306    private FTPFile parseFileList(final String entry) {
307        if (matches(entry)) {
308            final FTPFile file = new FTPFile();
309            file.setRawListing(entry);
310            final String name = group(2);
311            final String dsorg = group(1);
312            file.setName(name);
313
314            // DSORG
315            if ("PS".equals(dsorg)) {
316                file.setType(FTPFile.FILE_TYPE);
317            }
318            else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) {
319                // regex already ruled out anything other than PO or PO-E
320                file.setType(FTPFile.DIRECTORY_TYPE);
321            }
322            else {
323                return null;
324            }
325
326            return file;
327        }
328
329        return null;
330    }
331
332    /**
333     * Parse entries within a partitioned dataset.
334     *
335     * Format of a memberlist within a PDS:
336     * <pre>
337     *    0         1        2          3        4     5     6      7    8
338     *   Name      VV.MM   Created       Changed      Size  Init   Mod   Id
339     *   TBSHELF   01.03 2002/09/12 2002/10/11 09:37    11    11     0 KIL001
340     *   TBTOOL    01.12 2002/09/12 2004/11/26 19:54    51    28     0 KIL001
341     *
342     * -------------------------------------------
343     * [1] Name
344     * [2] VV.MM: Version . modification
345     * [3] Created: yyyy / MM / dd
346     * [4,5] Changed: yyyy / MM / dd HH:mm
347     * [6] Size: number of lines
348     * [7] Init: number of lines when first created
349     * [8] Mod: number of modified lines a last save
350     * [9] Id: User id for last update
351     * </pre>
352     *
353     * @param entry zosDirectoryEntry
354     * @return null: entry was not parsed.
355     */
356    private FTPFile parseMemberList(final String entry) {
357        final FTPFile file = new FTPFile();
358        if (matches(entry)) {
359            file.setRawListing(entry);
360            final String name = group(1);
361            final String datestr = group(2) + " " + group(3);
362            file.setName(name);
363            file.setType(FTPFile.FILE_TYPE);
364            try {
365                file.setTimestamp(super.parseTimestamp(datestr));
366            } catch (final ParseException e) {
367                // just ignore parsing errors.
368                // TODO check this is ok
369                // Drop thru to try simple parser
370            }
371            return file;
372        }
373
374        /*
375         * Assigns the name to the first word of the entry. Only to be used from a
376         * safe context, for example from a memberlist, where the regex for some
377         * reason fails. Then just assign the name field of FTPFile.
378         */
379        if (entry != null && !entry.trim().isEmpty()) {
380            file.setRawListing(entry);
381            final String name = entry.split(" ")[0];
382            file.setName(name);
383            file.setType(FTPFile.FILE_TYPE);
384            return file;
385        }
386        return null;
387    }
388
389    /**
390     * Matches these entries, note: no header:
391     * <pre>
392     * [1]      [2]      [3]   [4] [5]
393     * IBMUSER1 JOB01906 OUTPUT 3 Spool Files
394     * 012345678901234567890123456789012345678901234
395     *           1         2         3         4
396     * -------------------------------------------
397     * Group in regex
398     * [1] Job name
399     * [2] Job number
400     * [3] Job status (INPUT,ACTIVE,OUTPUT)
401     * [4] Number of sysout files
402     * [5] The string "Spool Files"
403     *</pre>
404     *
405     * @param entry zosDirectoryEntry
406     * @return null: entry was not parsed.
407     */
408    private FTPFile parseJeslevel1List(final String entry) {
409        if (matches(entry)) {
410            final FTPFile file = new FTPFile();
411            if (group(3).equalsIgnoreCase("OUTPUT")) {
412                file.setRawListing(entry);
413                final String name = group(2); /* Job Number, used by GET */
414                file.setName(name);
415                file.setType(FTPFile.FILE_TYPE);
416                return file;
417            }
418        }
419
420        return null;
421    }
422
423    /**
424     * Matches these entries:
425     * <pre>
426     * [1]      [2]      [3]     [4]    [5]
427     * JOBNAME  JOBID    OWNER   STATUS CLASS
428     * IBMUSER1 JOB01906 IBMUSER OUTPUT A       RC=0000 3 spool files
429     * IBMUSER  TSU01830 IBMUSER OUTPUT TSU     ABEND=522 3 spool files
430     * 012345678901234567890123456789012345678901234
431     *           1         2         3         4
432     * -------------------------------------------
433     * Group in regex
434     * [1] Job name
435     * [2] Job number
436     * [3] Owner
437     * [4] Job status (INPUT,ACTIVE,OUTPUT)
438     * [5] Job Class
439     * [6] The rest
440     * </pre>
441     *
442     * @param entry zosDirectoryEntry
443     * @return null: entry was not parsed.
444     */
445    private FTPFile parseJeslevel2List(final String entry) {
446        if (matches(entry)) {
447            final FTPFile file = new FTPFile();
448            if (group(4).equalsIgnoreCase("OUTPUT")) {
449                file.setRawListing(entry);
450                final String name = group(2); /* Job Number, used by GET */
451                file.setName(name);
452                file.setType(FTPFile.FILE_TYPE);
453                return file;
454            }
455        }
456
457        return null;
458    }
459
460    /**
461     * preParse is called as part of the interface. Per definition is is called
462     * before the parsing takes place.
463     * Three kind of lists is recognize:
464     * z/OS-MVS File lists
465     * z/OS-MVS Member lists
466     * unix file lists
467     * @since 2.0
468     */
469    @Override
470    public List<String> preParse(final List<String> orig) {
471        // simply remove the header line. Composite logic will take care of the
472        // two different types of
473        // list in short order.
474        if (orig != null && !orig.isEmpty()) {
475            final String header = orig.get(0);
476            if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) {
477                setType(FILE_LIST_TYPE);
478                super.setRegex(FILE_LIST_REGEX);
479            } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) {
480                setType(MEMBER_LIST_TYPE);
481                super.setRegex(MEMBER_LIST_REGEX);
482            } else if (header.indexOf("total") == 0) {
483                setType(UNIX_LIST_TYPE);
484                unixFTPEntryParser = new UnixFTPEntryParser();
485            } else if (header.indexOf("Spool Files") >= 30) {
486                setType(JES_LEVEL_1_LIST_TYPE);
487                super.setRegex(JES_LEVEL_1_LIST_REGEX);
488            } else if (header.indexOf("JOBNAME") == 0
489                    && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS
490                setType(JES_LEVEL_2_LIST_TYPE);
491                super.setRegex(JES_LEVEL_2_LIST_REGEX);
492            } else {
493                setType(UNKNOWN_LIST_TYPE);
494            }
495
496            if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary
497                orig.remove(0);
498            }
499        }
500
501        return orig;
502    }
503
504    /**
505     * Explicitly set the type of listing being processed.
506     * @param type The listing type.
507     */
508    void setType(final int type) {
509        isType = type;
510    }
511
512    /*
513     * @return
514     */
515    @Override
516    protected FTPClientConfig getDefaultConfiguration() {
517        return new FTPClientConfig(FTPClientConfig.SYST_MVS,
518                DEFAULT_DATE_FORMAT, null);
519    }
520
521}