diff options
author | Joe Hunkeler <jhunkeler@gmail.com> | 2015-08-11 16:51:37 -0400 |
---|---|---|
committer | Joe Hunkeler <jhunkeler@gmail.com> | 2015-08-11 16:51:37 -0400 |
commit | 40e5a5811c6ffce9b0974e93cdd927cbcf60c157 (patch) | |
tree | 4464880c571602d54f6ae114729bf62a89518057 /sys/fmio | |
download | iraf-osx-40e5a5811c6ffce9b0974e93cdd927cbcf60c157.tar.gz |
Repatch (from linux) of OSX IRAF
Diffstat (limited to 'sys/fmio')
43 files changed, 3170 insertions, 0 deletions
diff --git a/sys/fmio/README b/sys/fmio/README new file mode 100644 index 00000000..726dbc3b --- /dev/null +++ b/sys/fmio/README @@ -0,0 +1,339 @@ +FMIO -- BINARY FILE MANAGER (Jul88 DCT) +---------------------------------------------- + + This directory contains the sources for a general low level binary file +manager (FMIO). The purpose of this file manager is to manage a fixed number +of "lightweight files", or LFILES, maintained within a single variable length +binary file. This facility is used by higher level interfaces such as +database interfaces to store variable length objects efficiently in a single +host binary file. + + +1. INTERFACE PROCEDURES + +1.1 General Procedures + + The main FMIO interface procedures are summarized in the table below. +Most access to lfile data is intended to be via the FIO binary file driver +procedures (beginning with fm_lfopen in the figure). + + yes|no = fm_acccess (datafile, mode) + fm_rename (datafile, newname) + fm_copy (datafile, newname) + fm_delete (datafile) + fm_rebuild (datafile) + + fm = fm_open (datafile, mode) + fm_seti (fm, param, ival) + ival = fm_stati (fm, param) + fm_debug (fm, out, what) + fm_copyo (fm, fm_to) + fm_sync (fm) + fm_close (fm) + + lfile = fm_nextlfile (fm) + fm_lfname (fm, lfile, type, lfname, maxch) + ERR|OK = fm_lfparse (lfname, fm, lfile, type) + fm_lfcopy (fm_src, lfile_src, fm_dst, lfile_dst) + fm_fopen (fm, lfile, mode, type) + + fm_lfopen (lfname, mode, lf) + fm_lfstati (lf, param, ival) + fm_lfaread (lf, buf, nbytes, offset, status) + fm_lfawrite (lf, buf, nbytes, offset, status) + fm_lfawait (lf, status) + fm_lfclose (lf, status) + + fm_lfstat (fm, lfile, statbuf) + fm_lfdelete (fm, lfile) + fm_lfundelete (fm, lfile) + + +1.2 Buffer Cache + + To avoid excessive disk i/o when randomly accessing the datafile, it is +desirable to maintain a cache of several lfile data buffers, e.g., so that +accesses to a series of objects stored in a single lfile, or repeated accesses +to portions of several lfiles should incur minimal disk accesses. A simple +way to implement such a buffer cache is to simply open each lfile as a file +under FIO, leaving it up to FIO to manage the file buffer, and maintaining +a LRU cache of open lfiles. The number of buffers (open lfiles) is easily +parameterized. The buffer cache procedures are summarized in the figure +below. + + fd = fm_getfd (fm, lfile, mode, type) + fm_retfd (fm, lfile) + fm_lockout (fm, lfile) + fm_debugfd (fm, out) + +The fm_getfd routine maps an lfile onto a file descriptor. A file descriptor +is opened on the lfile only when necessary. Once opened, an lfile remains in +the cache until forced out by the LRU replacement algorithm, or the datafile +is closed. While the datafile remains open, removal of an lfile from the +cache (closing the associated file descriptor) is permitted only after a call +to fm_retfd; calling this routine does not immediately close the file, it only +permits it to be closed. Repeated to fm_getfd should return a file descriptor +immediately, with very little overhead, with an already active file buffer, +hence repeated calls to the cache manager and FIO may often be made without +incurring any disk accesses. + +Note that lfiles may be opened on file descriptors via direct calls to the +file manager, regardless of whether these lfiles are already open in the +buffer cache (e.g., with fm_fopen). This allows two or more independent +file buffers to be simultaneously active on the same lfile, but opens the +possibility of loss of data if the buffers overlap. If this is a problem, +the routine fm_lockoutfd may be called to prevent inadvertent use of an lfile +by the cache. This should be followed by a call to fm_retfd to clear the +lockout bit once the reason for the lockout (usually a noncached lfile open) +is gone. The routine fm_debugfd will print information on stream 'out' +describing the status of the buffer cache. + + +2. FILE STRUCTURE + +The layout of a datafile is as follows: + + + +-------------------------+ + | | datafile header | (fixed size) + stored | | +-------------------+ | + as - + | file table | (configurable) + unit | | +-------------------+ | + | | page table index | (configurable) + + +-------------------------+ + | + data pages (dynamic) + | + v + +The datafile header is a fixed format binary structure. The file table +contains one entry for each lfile stored in the datafile; the maximum number +of lfiles is fixed at datafile creation time. Each lfile is known by its +lfile number, ranging from zero to MAXLFILES. Lfile zero is the PAGE TABLE, +used to map each data page in the datafile to the lfile to which it is +allocated. The first user lfile is hence number 1. Lfiles may by any size; +storage is allocated in units of PAGES. The page size is fixed at datafile +creation time. There are two types of files, binary (opaque) files, and +text files. Both file types appear as binary files to the high level code, +i.e., both are accessed by a FIO binary file driver, the only difference +being that for a text file, data blocks are assumed to contain text and are +packed/unpacked during i/o (saving storage and rendering the file machine +independent). + +It is important to realize that lfiles are referred to only by FILE NUMBER +in this interface; any association with symbolic names must be made at a +higher level (lfiles are by no means necessarily associated with "files" at +the higher level, i.e., they might be used to store variable length parameters, +relations, or whatever). All lfiles exist, in a sense, as zero length files +at datafile creation time. To open a new lfile, one first calls fm_nextlfile +to get the file number of an empty lfile. Lfiles can be deleted, but storage +is never deallocated; new pages are always allocated at the end of file. +Hence deleted lfiles can be undeleted, and the entire datafile must normally +be copied (or "rebuilt") to reclaim unused space and coalesce file segments +for more efficient i/o. (There are cases where a deleted lfile can be reused +without rebuilding the lfile: fm_nextlfile will begin reusing deleted lfiles +after it wraps around, and the client software can always open an lfile +NEW_FILE, overwriting the pages already allocated to the lfile). + +The FMIO datafile itself, and any text files stored therein, is maintained in +a machine indepenent format. Binary file data is merely copied to and from +the datafile, hence it is up to the client software to store binary data in +a machine independent format, if desired. + + +2.1 Recovery + + Since new data pages are always allocated at the end of file (next +available PTE), and the datafile state is always sync-ed as a unit, protected +as a critical section (ignoring modifications to lfile data), a datafile +should always be recoverable after a crash, with loss only of data written +since the last sync. The datafile is sync-ed automatically every several +minutes. Applications wishing to protect newly written lfile data can sync +the datafile manually if desired. + + +3. RUNTIME DATA STRUCTURES + +The internal runtime data structures are summarized below. The terminology +used is as follows: + + FM file manager + FT file table + FTE file table entry + LFILE lightweight file + PAGE unit of datafile file storage + PT page table + PTE page table entry + PTI page table index + + +# FMDES -- Main FM descriptor. +struct fmdes { + int fm_magic # identifies file/descriptor type + int fm_active # set once descriptor is initialized + int fm_chan # host i/o channel for datafile + int fm_mode # datafile access mode + int fm_dfversion # datafile file version + int fm_szpage # datafile page size, bytes + int fm_nlfiles # number of lfiles + int fm_datastart # file offset of first data page + int fm_devblksize # device block size + int fm_optbufsize # default file buffer size + int fm_maxbufsize # maximum file buffer size + int fm_lsynctime # time descriptor last updated on disk + int fm_dhmodified # set if header needs to be updated + + int fm_ftoff # offset (su) of FT in datafile + int fm_ftlastnf # file number of last lfile allocated + struct fte (*fm_ftable)[] # file table storage + + int fm_ptioff # offset (su) of PTI in datafile + int fm_ptilen # allocated length of PTI + int fm_ptinpti # number of PTI entries in use + int fm_ptindex[] # PTI storage + + int fm_ptlen # allocated length of page table array + int fm_ptnpte # number of PTE's in use (#data pages) + int fm_ptlupte # highest PTE updated on disk + short fm_ptable[] # runtime page table array + + struct lfcache *fm_lfcache; # lfile cache descriptor + int fm_errcode # error code of posted error + char fm_erropstr[] # operand string of posted error + char fm_dfname[] # datafile name, for error messages +} + + +3.1 Page Table + + During runtime access to the datafile, the page table is a vector mapping +each datafile page to an lfile. Each page is allocated to a single lfile, and +lfile storage is allocated in units of pages. As the datafile is extended by +writing to lfiles, elements are the in-core page table array. + +When an lfile is first accessed, the in-core page table is scanned to find +those pages belonging to the lfile, building up a vector mapping offsets in +the lfile into datafile page numbers, i.e., to offsets in the physical +datafile. As new pages are allocated to an lfile by writing at end of file, +both the lfile page vector and datafile page table are extended. + +Assume the datafile page size is 512 bytes. Since a PTE is 2 bytes, each PT +page holds 256 PTEs, representing 128 Kb of file space. 1 Mb of file space +(2048 pages) therefore requires 8 pages of page table space. If we allocate +a default PT index of 256 slots, this gives us (for a 512 byte page) a 32 Mb +default maximum file size. + +Only the PT index, stored in the datafile header, and the PT pages mapping +datafile page to lfile, are physically stored in the datafile. The PT index +size is fixed. The PT pages may be stored anywhere in the data pages and +are pointed to by the PT index. The PT is stored as lfile zero. + +This scheme is not intended for use with extremely large datafiles, or with +datafiles containing a very large number of lfiles. A datafile of 32-256 Mb, +page size 512-4096, containing up to several hundred lfiles is the design +limit. Of course, each datafile is a single host file, and there can be any +number of datafiles. + + +3.2 File Table + + The file table (FT) describes each lfile in the datafile. Each lfile +has an entry in the file table, regardless of whether the lfile has ever been +accessed or contains any data. As stored in the datafile, the FT is an array +of file table entries (FTEs) containing two longwords of data each, i.e., + + FTE.1: file size, bytes + FTE.2: flag bits (text, deleted, etc.) + +Additional information must be maintained while an lfile is being accessed +at runtime. The full runtime FTE is as follows. + + struct fte { + struct fmdes *lf_fm # backpointer to FMIO descriptor + int lf_fsize # file size, bytes + int lf_flags # flag bits + int lf_status # runtime i/o status (byte count) + int lf_ltsize # logical byte size of last transfer + int lf_npages # npages in lfile (pagemap size) + int *lf_pagemap # pagemap array for lfile + int lf_pmlen # pagemap array length (allocated) + } + +flag bits: + + LF_DELETED set if the lfile is deleted + LF_TEXTFILE set if the lfile data is byte-packed text + LF_IOINPROGRESS set when i/o transfer is in progress + +Deleting an lfile merely causes the FT_DELETED bit to be set; the actual +data will be lost only if the lfile is reused or explicitly overwritten, +or in a copy or rebuild operation. Lfile space, once allocated, is never +freed, i.e., new pages are always allocated at the end of the datafile, +but lfile space can be *reused* by opening the lfile NEW_FILE and writing +into it. + +Space for all lfile descriptors is preallocated in the FTE array at datafile +open time. When an lfile is first accessed the pagemap for that lfile is +filled in; subsequent accesses to the lfile (lfile opens) while the datafile +remains open require very little overhead, since the lfile descriptor is +accessible via a vectored array reference, and will already have been activated. + + +4. EXAMPLE + + The following dialogue is from the debug tasks in ZZDEBUG. A datafile Q +containing four textfiles has been created, and we print out the contents of +this with SHOW. Next we copy the datafile to Q, and "show" that. Note that +the page table file is moved to the beginning of the data pages area (which +will avoid an extra disk access during the datafile open), and all lfiles +have been rendered contiguous (lfile 5 was not stored in two segments in the +original datafile). + +> ? + create wfile pfile show copy rebuild +> +> show +datafile: q +FMIO V1.1: datafile=q, pagesize=512, nlfiles=128 +nlfinuse=5, nlfdeleted=0, nlffree=123, ftoff=13, ftlastnf=0 +headersize=2560, filesize=20992, freespace=1408 bytes (6%) +fm=15AE9X, chan=4, mode=2, time since last sync=1 seconds +datastart=2561, devblksize=512, optbufsize=2048, maxbufsize=0 +ptioff=271, ptilen=256, npti=1, ptlen=512, npte=36, lupte=36 +====================== file table ======================= + 0 size=72 [page table] + 1 size=1114 textfile + 2 size=7037 textfile + 5 size=4422 textfile + 6 size=4379 textfile +=================== page table index ==================== + 4 +====================== page table ======================= + 1 1 1 0 2 2 2 2 2 2 2 2 2 2 2 + 2 2 2 5 5 6 6 6 6 6 6 6 6 6 5 + 5 5 5 5 5 5 +> +> copy +source: q +destination: p +> +> show +datafile: p +FMIO V1.1: datafile=p, pagesize=512, nlfiles=128 +nlfinuse=5, nlfdeleted=0, nlffree=123, ftoff=13, ftlastnf=0 +headersize=2560, filesize=20992, freespace=1408 bytes (6%) +fm=15AE9X, chan=4, mode=2, time since last sync=0 seconds +datastart=2561, devblksize=512, optbufsize=2048, maxbufsize=0 +ptioff=271, ptilen=256, npti=1, ptlen=512, npte=36, lupte=36 +====================== file table ======================= + 0 size=72 [page table] + 1 size=1114 textfile + 2 size=7037 textfile + 5 size=4422 textfile + 6 size=4379 textfile +=================== page table index ==================== + 1 +====================== page table ======================= + 0 1 1 1 2 2 2 2 2 2 2 2 2 2 2 + 2 2 2 5 5 5 5 5 5 5 5 5 6 6 6 + 6 6 6 6 6 6 +> diff --git a/sys/fmio/fmaccess.x b/sys/fmio/fmaccess.x new file mode 100644 index 00000000..c1324fc4 --- /dev/null +++ b/sys/fmio/fmaccess.x @@ -0,0 +1,15 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +# FM_ACCESS -- Test to see if the name FMIO datafile exists and is accessible +# with the given permissions. + +int procedure fm_access (dfname, mode) + +char dfname[ARB] #I datafile name +int mode #I access mode (0 to just test existence) + +int access() + +begin + return (access (dfname, mode, 0)) +end diff --git a/sys/fmio/fmclose.x b/sys/fmio/fmclose.x new file mode 100644 index 00000000..87e20674 --- /dev/null +++ b/sys/fmio/fmclose.x @@ -0,0 +1,51 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <error.h> +include <knet.h> +include "fmio.h" + +# FM_CLOSE -- Close a datafile opened under FMIO. + +procedure fm_close (fm) + +pointer fm #I FMIO descriptor + +pointer lf +int status, i +errchk fmio_bind, fm_fcfree, fmio_errchk + +begin + # An open-new-file followed by a close should create an empty datafile. + if (FM_ACTIVE(fm) == NO) + call fmio_bind (fm) + + # Shut down the file cache, if in use (does a sync). + if (FM_FCACHE(fm) != NULL) + call fm_fcfree (fm) + else + call fm_sync (fm) + + # Report any posted errors. + call fmio_errchk (fm) + + # Close the physical datafile. + call zclsbf (FM_CHAN(fm), status) + if (status == ERR) + iferr (call syserrs (SYS_FMCLOSE, FM_DFNAME(fm))) + call erract (EA_WARN) + + # Free any storage used by the runtime file table. + lf = FM_FTABLE(fm) + do i = 0, FM_NLFILES(fm) { + if (LF_PAGEMAP(lf) != NULL) + call mfree (LF_PAGEMAP(lf), TY_INT) + lf = lf + LEN_FTE + } + + # Free the main descriptor. + call mfree (FM_PTABLE(fm), TY_SHORT) + call mfree (FM_PTINDEX(fm), TY_INT) + call mfree (FM_FTABLE(fm), TY_INT) + call mfree (fm, TY_STRUCT) +end diff --git a/sys/fmio/fmcopy.x b/sys/fmio/fmcopy.x new file mode 100644 index 00000000..1125a820 --- /dev/null +++ b/sys/fmio/fmcopy.x @@ -0,0 +1,37 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <error.h> +include "fmset.h" + +# FM_COPY -- Copy a datafile, preserving all the physical attributes, but +# eliminating waste storage and rendering file structures logically contiguous. + +procedure fm_copy (dfname, newname) + +char dfname[ARB] #I existing datafile +char newname[ARB] #I new datafile name + +pointer o_fm, n_fm +pointer fm_open() +int fm_stati() +errchk fm_open, fm_copyo + +begin + # Open the old and new datafiles. + o_fm = fm_open (dfname, READ_ONLY) + n_fm = fm_open (newname, NEW_FILE) + + # The child inherits the attributes of the parent. + call fm_seti (n_fm, FM_PAGESIZE, fm_stati(o_fm,FM_PAGESIZE)) + call fm_seti (n_fm, FM_MAXLFILES, fm_stati(o_fm,FM_MAXLFILES)) + + # Copy the datafile and clean up. + iferr (call fm_copyo (o_fm, n_fm)) { + call fm_close (o_fm) + call fm_close (n_fm) + call erract (EA_ERROR) + } else { + call fm_close (o_fm) + call fm_close (n_fm) + } +end diff --git a/sys/fmio/fmcopyo.x b/sys/fmio/fmcopyo.x new file mode 100644 index 00000000..a6fd0be2 --- /dev/null +++ b/sys/fmio/fmcopyo.x @@ -0,0 +1,63 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <mach.h> +include "fmio.h" + +# FM_COPYO -- Copy an existing, open datafile to a new, open but empty +# datafile, rendering all lfile storage contiguous and omitting deleted +# lfiles. Lfile numbers are preserved by the copy operation, hence +# deleted lfiles will leave holes (zero length lfiles) in the file table. + +procedure fm_copyo (old, new) + +pointer old, new #I FMIO descriptors of source and destination + +pointer o_ft, o_lf +int n_szbpage, nlfiles, dpages, npte_perpage, npti, p1, dp, i +errchk fmio_bind, syserrs, fm_lfcopy +int fmio_extend() + +begin + call fmio_bind (old) + call fmio_bind (new) + + # Scan the file table of the old datafile and determine the number of + # data pages required to store the valid lfiles therein. Note that + # the page size may differ in the old and new datafiles. + + o_ft = FM_FTABLE(old) + n_szbpage = FM_SZBPAGE(new) + nlfiles = min (FM_NLFILES(old), FM_NLFILES(new)) + dpages = 0 + + do i = 0, nlfiles { + o_lf = o_ft + i * LEN_FTE + if (LF_FSIZE(o_lf) <= 0 || and(LF_FLAGS(o_lf),LFF_ALLOCATED) == 0) + next + dpages = dpages + (LF_FSIZE(o_lf) + n_szbpage-1) / n_szbpage + } + + # Now allocate enough lfile 0 space in the new datafile to permit + # contiguous storage of the entire datafile page table. + + npte_perpage = n_szbpage / (SZ_SHORT * SZB_CHAR) + npti = (dpages + npte_perpage-1) / npte_perpage + if (npti > FM_PTILEN(new)) + call syserrs (SYS_FMPTIOVFL, FM_DFNAME(new)) + + for (p1=FM_PTINPTI(new)+1; p1 <= npti; p1=p1+1) { + dp = fmio_extend (new, PT_LFILE, 1) + if (dp == ERR) + call syserrs (SYS_FMCOPYO, FM_DFNAME(new)) + Memi[FM_PTINDEX(new)+p1-1] = dp + FM_PTINPTI(new) = p1 + FM_DHMODIFIED(new) = YES + } + + # Copy the lfiles (excluding the page table). + do i = 1, nlfiles + call fm_lfcopy (old, i, new, i) + + call fm_sync (new) +end diff --git a/sys/fmio/fmdebug.x b/sys/fmio/fmdebug.x new file mode 100644 index 00000000..183e4481 --- /dev/null +++ b/sys/fmio/fmdebug.x @@ -0,0 +1,182 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmset.h" +include "fmio.h" + +# FM_DEBUG -- Print debug info on the contents of a datafile. +# +# Flags: +# FMD_HEADER general header parameters +# FMD_FTABLE summarize file table contents +# FMD_PTINDEX print page table index +# FMD_PTABLE print page table +# FMD_ALL print everything + +procedure fm_debug (fm, out, what) + +pointer fm #I FMIO descriptor +int out #I output file +int what #I what to print + +pointer ft, lf +bool deleted +int nlfiles, nlfdeleted, nlfinuse, i +int szbpage, spaceinuse, filesize, freespace +long clktime() +errchk fmio_bind + +define end_header_ 91 +define end_ftable_ 92 +define end_ptindex_ 93 +define end_ptable_ 94 + +begin + call fmio_bind (fm) + + ft = FM_FTABLE(fm) + nlfiles = FM_NLFILES(fm) + szbpage = FM_SZBPAGE(fm) + + # Print header and summary information? + if (and (what, FMD_HEADER) == 0) + goto end_header_ + + # Scan the file table and compute some important statistics. + spaceinuse = 0 + nlfdeleted = 0 + nlfinuse = 0 + + do i = 0, nlfiles { + lf = ft + i * LEN_FTE + deleted = (and (LF_FLAGS(lf), LFF_DELETED) != 0) + if (deleted) + nlfdeleted = nlfdeleted + 1 + if (!deleted) { + if (LF_FSIZE(lf) > 0) + spaceinuse = spaceinuse + LF_FSIZE(lf) + if (and (LF_FLAGS(lf), LFF_ALLOCATED) != 0) + nlfinuse = nlfinuse + 1 + } + } + + filesize = FM_PTNPTE(fm) * szbpage + FM_DATASTART(fm) - 1 + freespace = max (0, filesize - spaceinuse - (FM_DATASTART(fm)-1)) + + call fprintf (out, + "FMIO V%d.%d: datafile=%s, pagesize=%d, nlfiles=%d\n") + call pargi (FM_DFVERSION(fm) / 100) + call pargi (mod(FM_DFVERSION(fm),100)) + call pargstr (FM_DFNAME(fm)) + call pargi (szbpage) + call pargi (nlfiles) + + call fprintf (out, + "nlfinuse=%d, nlfdeleted=%d, nlffree=%d, ftoff=%d, ftlastnf=%d\n") + call pargi (nlfinuse) + call pargi (nlfdeleted) + call pargi (nlfiles - nlfinuse) + call pargi (FM_FTOFF(fm)) + call pargi (FM_FTLASTNF(fm)) + + call fprintf (out, + "headersize=%d, filesize=%d, freespace=%d bytes (%d%%)\n") + call pargi (FM_DATASTART(fm) - 1) + call pargi (filesize) + call pargi (freespace) + if (freespace <= 0) + call pargi (0) + else + call pargi (freespace * 100 / filesize) + + call fprintf (out, + "fm=%xX, chan=%d, mode=%d, time since last sync=%d seconds\n") + call pargi (fm) + call pargi (FM_CHAN(fm)) + call pargi (FM_MODE(fm)) + call pargi (clktime (FM_LSYNCTIME(fm))) + + call fprintf (out, + "datastart=%d, devblksize=%d, optbufsize=%d, maxbufsize=%d\n") + call pargi (FM_DATASTART(fm)) + call pargi (FM_DEVBLKSIZE(fm)) + call pargi (FM_OPTBUFSIZE(fm)) + call pargi (FM_MAXBUFSIZE(fm)) + + call fprintf (out, "ptioff=%d, ptilen=%d, npti=%d, ") + call pargi (FM_PTIOFF(fm)) + call pargi (FM_PTILEN(fm)) + call pargi (FM_PTINPTI(fm)) + call fprintf (out, "ptlen=%d, npte=%d, lupte=%d\n") + call pargi (FM_PTLEN(fm)) + call pargi (FM_PTNPTE(fm)) + call pargi (FM_PTLUPTE(fm)) + +end_header_ + + # Print file table? + if (and (what, FMD_FTABLE) == 0) + goto end_ftable_ + + call fprintf (out, + "====================== file table =======================\n") + do i = 0, nlfiles { + lf = ft + i * LEN_FTE + if (LF_FSIZE(lf) == 0) + next + call fprintf (out, " %4d size=%d") + call pargi (i) + call pargi (LF_FSIZE(lf)) + if (i == 0) + call fprintf (out, " [page table]") + if (LF_PAGEMAP(lf) != NULL) { + call fprintf (out, " npages=%d pmlen=%d") + call pargi (LF_NPAGES(lf)) + call pargi (LF_PMLEN(lf)) + } + if (and (LF_FLAGS(lf), LFF_ALLOCATED) != 0) + call fprintf (out, " allocated") + if (and (LF_FLAGS(lf), LFF_DELETED) != 0) + call fprintf (out, " deleted") + if (and (LF_FLAGS(lf), LFF_TEXTFILE) != 0) + call fprintf (out, " textfile") + call fprintf (out, "\n") + } + +end_ftable_ + + # Print page table index? + if (and (what, FMD_PTINDEX) == 0) + goto end_ptindex_ + + call fprintf (out, + "=================== page table index ====================\n") + do i = 0, FM_PTINPTI(fm) - 1 { + call fprintf (out, " %4d") + call pargi (Memi[FM_PTINDEX(fm)+i]) + if (mod (i+1, 15) == 0) + call fprintf (out, "\n") + } + if (mod (FM_PTINPTI(fm), 15) != 0) + call fprintf (out, "\n") + +end_ptindex_ + + # Print page table? + if (and (what, FMD_PTABLE) == 0) + goto end_ptable_ + + call fprintf (out, + "====================== page table =======================\n") + do i = 0, FM_PTNPTE(fm) - 1 { + call fprintf (out, " %4d") + call pargs (Mems[FM_PTABLE(fm)+i]) + if (mod (i+1, 15) == 0) + call fprintf (out, "\n") + } + if (mod (FM_PTINPTI(fm), 15) != 0) + call fprintf (out, "\n") + +end_ptable_ + + call flush (out) +end diff --git a/sys/fmio/fmdelete.x b/sys/fmio/fmdelete.x new file mode 100644 index 00000000..b90fb2af --- /dev/null +++ b/sys/fmio/fmdelete.x @@ -0,0 +1,11 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +# FM_DELETE -- Delete a datafile. + +procedure fm_delete (dfname) + +char dfname[ARB] #I datafile name + +begin + call delete (dfname) +end diff --git a/sys/fmio/fmfcache.x b/sys/fmio/fmfcache.x new file mode 100644 index 00000000..6e1d16fe --- /dev/null +++ b/sys/fmio/fmfcache.x @@ -0,0 +1,395 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <error.h> +include <fset.h> +include "fmset.h" +include "fmio.h" + +.help fcache +.nf ------------------------------------------------------------------------- +FCACHE -- FMIO File (Buffer) Cache package. + + This package is used to manage a cache of lfile data buffers maintained on +a FMIO datafile. The number and size of data buffers is user configurable: +the number of buffers by a call to FM_SETI to set FM_FCACHESIZE, and the buffer +size by a call to FSETI to set the file buffer size. + + fm_fcinit (fm, cachesize) + fm_fcdebug (fm, out, what) + fm_fcsync (fm) + fm_fcfree (fm) + + fd = fm_getfd (fm, lfile, mode, type) + fm_retfd (fm, lfile) + fm_lockout (fm, lfile) + fm_unlock (fm, lfile) + bool = fm_locked (fm, lfile) + +The cache is a conventional LRU cache. FM_GETFD returns the file descriptor +of an open, cached file opened on an lfile, locking the file in the cache in +the process. A subsequent call to FM_RETFD is required to allow the cache +slot to be reused. FM_LOCKOUT may be used to lock a particular lfile out of +the cache; FM_UNLOCK releases the lock. +.endhelp -------------------------------------------------------------------- + +define LEN_FCACHE (3+($1)*LEN_FSLOT) # len = LEN_FCACHE(cachesize) +define FC_NFILES Memi[$1] # number of files in cache +define FC_REFCNT Memi[$1+1] # cache reference count +define FC_LFSTAT Memi[$1+2] # lfile statistics array +define FC_FS ((($2)-1)*LEN_FSLOT+($1)+3) # get slot pointer + +define LEN_FSLOT 4 +define FC_NREF Memi[$1] # number of refs to this slot +define FC_LRU Memi[$1+1] # LRU count for slot +define FC_LFILE Memi[$1+2] # lfile assigned to slot +define FC_FD Memi[$1+3] # file descriptor + + +# FM_GETFD -- Map a FIO file descriptor onto an lfile and return the file +# descriptor. The opened file descriptor remains in the cache after the +# access, until the file ages out of the cache. + +int procedure fm_getfd (fm, lfile, mode, type) + +pointer fm # FMIO descriptor +int lfile # lfile to be opened +int mode # file access mode +int type # file type + +int acmode, lru, i +pointer oldest, fc, fs, st + +bool fm_locked() +pointer fm_findlf() +int fm_fopen(), fstati() +errchk fm_fopen, fm_fcinit, syserrs, close +define ref_ 91 + +begin + fc = FM_FCACHE(fm) + if (fc == NULL) { + call fm_fcinit (fm, FM_SZFCACHE(fm)) + fc = FM_FCACHE(fm) + } + + # Keep debug statistics on lfile accesses. + st = FC_LFSTAT(fc) + if (st != NULL) + Mems[st+lfile] = Mems[st+lfile] + 1 + + fs = fm_findlf (fc, lfile) + if (fs != NULL) { + # If lfile is already in the cache and the new mode is NEW_FILE, + # or we need write perm and do not currently have it, close and + # reopen the lfile with the desired mode. + + if (mode == NEW_FILE || + (mode != READ_ONLY && fstati (FC_FD(fs), F_WRITE) == NO)) { +ref_ + if (FC_NREF(fs) > 1) + call syserrs (SYS_FMFSINUSE, FM_DFNAME(fm)) + if (FC_NREF(fs) == 1) + call close (FC_FD(fs)) + if (fm_locked (fm, lfile)) + call syserrs (SYS_FMLFLOCKED, FM_DFNAME(fm)) + + if (mode == WRITE_ONLY || mode == APPEND) + acmode = READ_WRITE + else + acmode = mode + + FC_FD(fs) = fm_fopen (fm, lfile, acmode, type) + FC_NREF(fs) = 1 + FC_LFILE(fs) = lfile + } + + # Reference the cached lfile. + FC_REFCNT(fc) = FC_REFCNT(fc) + 1 + FC_LRU(fs) = FC_REFCNT(fc) + FC_NREF(fs) = FC_NREF(fs) + 1 + if (mode == APPEND) + call seek (FC_FD(fs), EOFL) + + return (FC_FD(fs)) + + } else { + # Lfile is not in cache. Find the oldest slot. + oldest = NULL + lru = NULL + do i = 1, FC_NFILES(fc) { + fs = FC_FS(fc,i) + if (FC_NREF(fs) <= 1) + if (lru == NULL || FC_LRU(fs) < lru) { + lru = FC_LRU(fs) + oldest = fs + if (FC_NREF(fs) <= 0) + break + } + } + + # Abort if all cache slots are busy. + if (oldest == NULL) + call syserrs (SYS_FMFCFULL, FM_DFNAME(fm)) + + # Replace the file in the cache, and return new descriptor. + fs = oldest + goto ref_ + } +end + + +# FM_RETFD -- Return a cached file, i.e., decrement the reference count for +# the file so that it may be returned. If FM_RETFD is called when the file +# is sitting in the cache idle, the file is physically closed (i.e., a GETFD +# followed by one RETFD leaves the file cached and idle, ready for another +# GETFD without losing context, but another RETFD while the file is idle +# closes the file and removes it from the cache). + +procedure fm_retfd (fm, lfile) + +pointer fm #I FMIO descriptor +int lfile #I lfile to be returned + +pointer fc, fs +pointer fm_findlf() +errchk fm_fcinit + +begin + fc = FM_FCACHE(fm) + if (fc == NULL) { + call fm_fcinit (fm, FM_SZFCACHE(fm)) + fc = FM_FCACHE(fm) + } + + fs = fm_findlf (fc, lfile) + if (fs != NULL) { + FC_NREF(fs) = FC_NREF(fs) - 1 + if (FC_NREF(fs) <= 0) { + call close (FC_FD(fs)) + FC_NREF(fs) = 0 + } + } +end + + +# FM_LOCKOUT -- Lock an lfile out of the file cache. + +procedure fm_lockout (fm, lfile) + +pointer fm #I FMIO descriptor +int lfile #I lfile to be locked out + +pointer fc, fs, lf +pointer fm_findlf() +errchk fm_fcinit, close, syserrs + +begin + fc = FM_FCACHE(fm) + if (fc == NULL) { + call fm_fcinit (fm, FM_SZFCACHE(fm)) + fc = FM_FCACHE(fm) + } + + # Close lfile if it is already in the cache, but idle. + fs = fm_findlf (fc, lfile) + if (fs != NULL) { + if (FC_NREF(fs) > 1) + call syserrs (SYS_FMLOKACTLF, FM_DFNAME(fm)) + FC_NREF(fs) = FC_NREF(fs) - 1 + if (FC_NREF(fs) <= 0) { + call close (FC_FD(fs)) + FC_NREF(fs) = 0 + } + } + + # Set the lockout bit in the file table. + lf = FM_FTABLE(fm) + lfile * LEN_FTE + LF_FLAGS(lf) = or (LF_FLAGS(lf), LFF_LOCKOUT) +end + + +# FM_UNLOCK -- Unlock an lfile. + +procedure fm_unlock (fm, lfile) + +pointer fm #I FMIO descriptor +int lfile #I lfile to be locked out + +pointer lf + +begin + # Clear the lockout bit in the file table. + lf = FM_FTABLE(fm) + lfile * LEN_FTE + LF_FLAGS(lf) = and (LF_FLAGS(lf), not(LFF_LOCKOUT)) +end + + +# FM_LOCKED -- Test the lfile lockout bit. + +bool procedure fm_locked (fm, lfile) + +pointer fm #I FMIO descriptor +int lfile #I lfile to be locked out + +pointer lf + +begin + # Test the lockout bit in the file table. + lf = FM_FTABLE(fm) + lfile * LEN_FTE + return (and (LF_FLAGS(lf), LFF_LOCKOUT) != 0) +end + + +# FM_FINDLF -- Search the cache for the given lfile. + +pointer procedure fm_findlf (fc, lfile) + +pointer fc #I file cache descriptor +int lfile #I lfile to search for + +int i +pointer fs + +begin + do i = 1, FC_NFILES(fc) { + fs = FC_FS(fc,i) + if (FC_NREF(fs) != 0 && FC_LFILE(fs) == lfile) + return (fs) + } + + return (NULL) +end + + +# FM_FCDEBUG -- Print debug info describing the contents and status of the +# file cache. + +procedure fm_fcdebug (fm, out, what) + +pointer fm #I FMIO descriptor +int out #I output file +int what #I type of debug output + +int nref, i +pointer fc, fs, st +errchk fm_fcinit + +begin + fc = FM_FCACHE(fm) + if (fc == NULL) { + call fm_fcinit (fm, FM_SZFCACHE(fm)) + fc = FM_FCACHE(fm) + } + + # Print cache status. + if (and (what, FCD_CACHE) != 0) { + call fprintf (out, "# LRU NREF LFILE FD\n") + do i = 1, FC_NFILES(fc) { + fs = FC_FS(fc,i) + call fprintf (out, "%6d %4d %5d %3d\n") + call pargi (FC_LRU(fs)) + call pargi (FC_NREF(fs)) + if (FC_NREF(fs) > 0) { + call pargi (FC_LFILE(fs)) + call pargi (FC_FD(fs)) + } else { + call pargi (0) + call pargi (0) + } + } + } + + # Print lfile access statistics. + if (and (what, FCD_LFSTATISTICS) != 0) { + st = FC_LFSTAT(fc) + if (st != NULL) { + call fprintf (out, "-------- getfd's per lfile ---------\n") + do i = 0, FM_NLFILES(fm) { + nref = Mems[st+i] + if (nref > 0) { + call fprintf (out, "%4d %4d\n") + call pargi (i) + call pargi (nref) + } + } + } + } +end + + +# FM_FCINIT -- Init the file cache. + +procedure fm_fcinit (fm, cachesize) + +pointer fm #I FMIO descriptor +int cachesize #I size of cache, file slots + +pointer fc + +begin + if (FM_FCACHE(fm) != NULL) + call fm_fcfree (fm) + + call calloc (fc, LEN_FCACHE(cachesize), TY_STRUCT) + call calloc (FC_LFSTAT(fc), FM_NLFILES(fm)+1, TY_SHORT) + FC_NFILES(fc) = cachesize + FC_REFCNT(fc) = 1 + + FM_FCACHE(fm) = fc +end + + +# FM_FCFREE -- Shutdown the file cache and free all resources, ignoring +# all file reference counts. + +procedure fm_fcfree (fm) + +pointer fm #I FMIO descriptor + +int i +pointer fc, fs + +begin + fc = FM_FCACHE(fm) + if (fc == NULL) + return + + # Update the datafile. + call fm_fcsync (fm) + + # Close any cached files. + do i = 1, FC_NFILES(fc) { + fs = FC_FS(fc,i) + if (FC_NREF(fs) >= 1) + iferr (call close (FC_FD(fs))) + call erract (EA_WARN) + } + + call mfree (FC_LFSTAT(fc), TY_SHORT) + call mfree (FM_FCACHE(fm), TY_STRUCT) +end + + +# FM_FCSYNC -- Sync the file cache and datafile. + +procedure fm_fcsync (fm) + +pointer fm #I FMIO descriptor + +int i +pointer fc, fs + +begin + fc = FM_FCACHE(fm) + if (fc != NULL) { + do i = 1, FC_NFILES(fc) { + fs = FC_FS(fc,i) + if (FC_NREF(fs) >= 1) + iferr (call flush (FC_FD(fs))) + call erract (EA_WARN) + } + } + + call fm_sync (fm) +end diff --git a/sys/fmio/fmfopen.x b/sys/fmio/fmfopen.x new file mode 100644 index 00000000..6b3b897a --- /dev/null +++ b/sys/fmio/fmfopen.x @@ -0,0 +1,30 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +# FM_FOPEN -- Open an lfile as an FIO file. A FIO file descriptor is returned +# for the open file. CLOSE is used to close the opened file. + +int procedure fm_fopen (fm, lfile, mode, type) + +pointer fm #I FMIO descriptor +int lfile #I lfile to be opened +int mode #I file access mode +int type #I logical file type + +int fd +pointer sp, lfname +extern fm_lfopen(), fm_lfclose() +extern fm_lfaread(), fm_lfawrite(), fm_lfawait(), fm_lfstati() +int fopnbf() +errchk fopnbf + +begin + call smark (sp) + call salloc (lfname, SZ_FNAME, TY_CHAR) + + call fm_lfname (fm, lfile, type, Memc[lfname], SZ_FNAME) + fd = fopnbf (Memc[lfname], mode, fm_lfopen, fm_lfaread, fm_lfawrite, + fm_lfawait, fm_lfstati, fm_lfclose) + + call sfree (sp) + return (fd) +end diff --git a/sys/fmio/fmio.h b/sys/fmio/fmio.h new file mode 100644 index 00000000..41c43676 --- /dev/null +++ b/sys/fmio/fmio.h @@ -0,0 +1,97 @@ +# FMIO.H -- File manager interface definitions (private to FMIO). + +define FMIO_MAGIC 28006 # "fm" +define FMIO_VERSION 101 # current interface version number +define DEF_PAGESIZE 512 # default page size, bytes +define DEF_MAXLFILES 128 # default max lfiles / datafile +define DEF_MAXPTPAGES 256 # default max page table pages +define DEF_OPTBUFNP 4 # default npages in lfile FIO buffer +define DEF_MAXBUFNP 0 # default npages max buffer size +define DEF_BIGBUFNP 16 # large buffer for lfile copies +define DEF_DFHDRLEN 4096 # default DF header buflen (su) +define DEF_FCACHESIZE 8 # default open files in file cache +define DEF_PMLEN 64 # default lfile pagemap array length +define INC_PMLEN 128 # default page table buflen (pte) +define SZ_DFNAME 63 # datafile name +define SZ_ERROPSTR 63 # operand string, for posted errors +define SYNC_INTERVAL 300 # interval between automatic syncs +define PT_LFILE 0 # lfile in which page table is stored + +# Main FMIO descriptor. +define LEN_FMDES 150 +# GEN +define FM_MAGIC Memi[$1+0] # set once descriptor is initialized +define FM_ACTIVE Memi[$1+1] # set once descriptor is initialized +define FM_CHAN Memi[$1+2] # host channel of datafile +define FM_MODE Memi[$1+3] # access mode of datafile +define FM_DFVERSION Memi[$1+4] # datafile version number +define FM_SZBPAGE Memi[$1+5] # datafile page size, bytes +define FM_NLFILES Memi[$1+6] # number of lfiles in datafile +define FM_DATASTART Memi[$1+7] # file offset of first data page +define FM_DEVBLKSIZE Memi[$1+8] # device block size +define FM_OPTBUFSIZE Memi[$1+9] # optimum FIO buffer size +define FM_MAXBUFSIZE Memi[$1+10] # maximum FIO buffer size +define FM_SZFCACHE Memi[$1+11] # number of LF in file cache +define FM_LSYNCTIME Memi[$1+12] # time of last sync +define FM_DHMODIFIED Memi[$1+13] # datafile header has been modified +# FT +define FM_FTOFF Memi[$1+15] # offset of stored file table (su) +define FM_FTLASTNF Memi[$1+16] # last new file allocated (runtime) +define FM_FTABLE Memi[$1+17] # pointer to in-core file table +# PTI +define FM_PTIOFF Memi[$1+20] # offset of stored PTI (su) +define FM_PTILEN Memi[$1+21] # page table index length (allocated) +define FM_PTINPTI Memi[$1+22] # number of page table pages +define FM_PTINDEX Memi[$1+23] # pointer to page table index +# PT +define FM_PTLEN Memi[$1+25] # number of allocated PTE entries +define FM_PTNPTE Memi[$1+26] # number of PTEs (data pages) in-core +define FM_PTLUPTE Memi[$1+27] # last updated PTE on disk +define FM_PTABLE Memi[$1+28] # pointer to page table data +# LFC +define FM_FCACHE Memi[$1+30] # pointer to file cache descriptor +define FM_ERRCODE Memi[$1+31] # opcode for posting errors +define FM_ERROPSTR Memc[P2C($1+32)]# error operand string +define FM_DFNAME Memc[P2C($1+96)]# datafile name, for error messages + +# File table entry (FTE) during datafile access. +define LEN_FTE 8 # length of file table entry (ints) +define LF_FM Memi[$1] # backpointer to FMIO descriptor +define LF_FSIZE Memi[$1+1] # file size, bytes +define LF_FLAGS Memi[$1+2] # file bitflags +define LF_STATUS Memi[$1+3] # status word for async i/o +define LF_LTSIZE Memi[$1+4] # logical size of last transfer +define LF_NPAGES Memi[$1+5] # number of pages in file page table +define LF_PAGEMAP Memi[$1+6] # pointer to pagemap array +define LF_PMLEN Memi[$1+7] # length of pagemap array + +# FTE bitflags. +define LFF_SAVE 007B # flags saved while datafile is closed +define LFF_ALLOCATED 001B # lfile slot has been allocated +define LFF_DELETED 002B # set if file is deleted +define LFF_TEXTFILE 004B # set if file is a text file +define LFF_IOINPROGRESS 010B # set when i/o is in progress +define LFF_LOCKOUT 020B # set when i/o is in progress + +# -------------- +# Physical datafile file format. + +# Datafile file header. +define LEN_DHSTRUCT 12 +define DH_MAGIC Memi[$1] # magic value identifying data format +define DH_DFVERSION Memi[$1+1] # FMIO version used to write datafile +define DH_SZBPAGE Memi[$1+2] # datafile page size, bytes +define DH_NLFILES Memi[$1+3] # number of lfiles in datafile +define DH_FTOFF Memi[$1+4] # offset of file table in datafile +define DH_FTLASTNF Memi[$1+5] # last new file allocated (runtime) +define DH_PTIOFF Memi[$1+6] # offset of stored page table index +define DH_PTILEN Memi[$1+7] # page table index length (allocated) +define DH_PTINPTI Memi[$1+8] # number of page table pages +define DH_PTLEN Memi[$1+9] # number of allocated PTE entries +define DH_PTNPTE Memi[$1+10] # number of PTEs (data pages) +define DH_DATASTART Memi[$1+11] # file offset of first data page + +# File table entry (FTE) on disk. +define LEN_FTEX 2 # length of file table entry (ints) +define FT_FSIZE Memi[$1] # file size, bytes +define FT_FLAGS Memi[$1+1] # file bitflags diff --git a/sys/fmio/fmiobind.x b/sys/fmio/fmiobind.x new file mode 100644 index 00000000..b31c7ada --- /dev/null +++ b/sys/fmio/fmiobind.x @@ -0,0 +1,61 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <mach.h> +include "fmio.h" + +# FMIO_BIND -- Called when a new datafile is being created, to bind the +# current datafile parameters set in the descriptor to the physical datafile. + +procedure fmio_bind (fm) + +pointer fm #I FMIO descriptor + +pointer ft, pti, pt +int chan, szbpage, ftoff, ftlen, ptioff, ptilen, ptlen + +begin + if (FM_ACTIVE(fm) != NO) + return + + # Initialize buffer sizes. + call fmio_setbuf (fm) + + chan = FM_CHAN(fm) + szbpage = FM_SZBPAGE(fm) + ftoff = LEN_DHSTRUCT + 1 + ftlen = (FM_NLFILES(fm) + 1) * LEN_FTE + ptioff = ftoff + (FM_NLFILES(fm) + 1) * LEN_FTEX + ptilen = FM_PTILEN(fm) + ptlen = szbpage / (SZ_SHORT * SZB_CHAR) + + # Determine the byte offset of the first data page. + FM_DATASTART(fm) = max (1, + (((ptioff+ptilen-1) * SZ_INT*SZB_CHAR) + szbpage-1) / + szbpage) * szbpage + 1 + + # Initialize the file table. + call calloc (ft, ftlen, TY_STRUCT) + FM_FTOFF(fm) = ftoff + FM_FTLASTNF(fm) = 0 + FM_FTABLE(fm) = ft + + # Initialize the page table index. + call calloc (pti, ptilen, TY_INT) + FM_PTIOFF(fm) = ptioff + FM_PTINPTI(fm) = 0 + FM_PTINDEX(fm) = pti + + # Initialize the page table, stored in the data pages. Note that the + # page table length must be an integral multiple of the page size. + + call calloc (pt, ptlen, TY_SHORT) + FM_PTLEN(fm) = ptlen + FM_PTNPTE(fm) = 0 + FM_PTLUPTE(fm) = 0 + FM_PTABLE(fm) = pt + + FM_ACTIVE(fm) = YES + FM_DHMODIFIED(fm) = YES + + call fm_sync (fm) +end diff --git a/sys/fmio/fmioerr.x b/sys/fmio/fmioerr.x new file mode 100644 index 00000000..dd15e7be --- /dev/null +++ b/sys/fmio/fmioerr.x @@ -0,0 +1,20 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmio.h" + +# FMIO_ERRCHK -- Check for a posted error, and process the error if one is +# posted. + +procedure fmio_errchk (fm) + +pointer fm #I FMIO descriptor + +int errcode + +begin + errcode = FM_ERRCODE(fm) + if (errcode != OK) { + FM_ERRCODE(fm) = OK + call syserrs (errcode, FM_ERROPSTR(fm)) + } +end diff --git a/sys/fmio/fmioextnd.x b/sys/fmio/fmioextnd.x new file mode 100644 index 00000000..0517eb93 --- /dev/null +++ b/sys/fmio/fmioextnd.x @@ -0,0 +1,82 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <mach.h> +include "fmio.h" + +# FMIO_EXTEND -- Allocate new pages in the datafile. Allocate the new pages +# and modify the in-core page table and lfile pagemap accordingly. The PTI is +# not modified, allocating new pages to store the page table itself, until +# fm_sync is called. The page number of the first page allocated is returned +# as the function value (a contiguous sequence of pages will be allocated). + +int procedure fmio_extend (fm, lfile, npages) + +pointer fm #I FMIO descriptor +int lfile #I lfile getting the new pages +int npages #I number of pages to add + +pointer pt, pm, lf +int npte_perpage, npti +int inc, np, p1, p2, l1, l2, i +int krealloc() + +begin + # Extend the global page table. + p1 = FM_PTNPTE(fm) + 1 + p2 = p1 + npages - 1 + + # Make sure we have enough page table index entries for the new pages. + if (lfile != PT_LFILE) { + npte_perpage = FM_SZBPAGE(fm) / (SZB_CHAR*SZ_SHORT) + np = p2 + FM_PTILEN(fm) + npti = (np + npte_perpage-1) / npte_perpage + if (npti > FM_PTILEN(fm)) { + call fmio_posterr (fm, SYS_FMPTIOVFL, FM_DFNAME(fm)) + return (ERR) + } + } + + # Increase the size of the in-core page table if necessary. + if (p2 > FM_PTLEN(fm)) { + inc = FM_SZBPAGE(fm) / (SZ_SHORT * SZB_CHAR) + FM_PTLEN(fm) = ((p2 + inc-1) / inc) * inc + if (krealloc (FM_PTABLE(fm), FM_PTLEN(fm), TY_SHORT) == ERR) + return (ERR) + } + + # Add the pages to the global page table. + FM_PTNPTE(fm) = p2 + pt = FM_PTABLE(fm) + do i = p1, p2 + Mems[pt+i-1] = lfile + + lf = FM_FTABLE(fm) + lfile * LEN_FTE + pm = LF_PAGEMAP(lf) + + # Extend the lfile page table if the lfile is active. + if (pm != NULL) { + l1 = LF_NPAGES(lf) + 1 + l2 = l1 + npages - 1 + + # Increase the size of the lfile pagemap if necessary. + if (l2 > LF_PMLEN(lf)) { + LF_PMLEN(lf) = (l2 + INC_PMLEN-1) / INC_PMLEN * INC_PMLEN + if (krealloc (LF_PAGEMAP(lf), LF_PMLEN(lf), TY_INT) == ERR) + return (ERR) + pm = LF_PAGEMAP(lf) + } + + # Add the pages to the lfile page table. + LF_NPAGES(lf) = l2 + do i = l1, l2 + Memi[pm+i-1] = p1 + i - l1 + } + + # Update the FTE for lfile zero (the page table file). + lf = FM_FTABLE(fm) + LF_FSIZE(lf) = FM_PTNPTE(fm) * SZ_SHORT * SZB_CHAR + + FM_DHMODIFIED(fm) = YES + return (p1) +end diff --git a/sys/fmio/fmiopost.x b/sys/fmio/fmiopost.x new file mode 100644 index 00000000..23c5eaae --- /dev/null +++ b/sys/fmio/fmiopost.x @@ -0,0 +1,20 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmio.h" + +# FMIO_POSTERR -- Post an error. This is called to flag an error condition +# by low level code that cannot call the error handling code directly. + +procedure fmio_posterr (fm, errcode, opstr) + +pointer fm #I FMIO descriptor +int errcode #I error code +char opstr[ARB] #I operand id string + +begin + # In case of multiple errors, post only the first one. + if (FM_ERRCODE(fm) == OK) { + FM_ERRCODE(fm) = errcode + call strcpy (opstr, FM_ERROPSTR(fm), SZ_ERROPSTR) + } +end diff --git a/sys/fmio/fmiorhdr.x b/sys/fmio/fmiorhdr.x new file mode 100644 index 00000000..1b7f6c3c --- /dev/null +++ b/sys/fmio/fmiorhdr.x @@ -0,0 +1,147 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <mach.h> +include <knet.h> +include "fmio.h" + +# FMIO_READHEADER -- Read the header of a FMIO datafile and set up the FMIO +# runtime descriptor. + +procedure fmio_readheader (fm) + +pointer fm #I FMIO descriptor + +pointer sp, buf, dh, pti, pt, ft, ip, op +int offset, buflen, npti, p1, p2, d1, d2, b_off1, b_off2, i +int status, chan, nbytes, nwords, maxpages, szbpage +errchk syserrs, calloc, malloc, realloc +long clktime() + +begin + call smark (sp) + call salloc (dh, LEN_DHSTRUCT, TY_STRUCT) + + chan = FM_CHAN(fm) + + # Make a guess at the size of buffer needed to hold the header. + buflen = DEF_DFHDRLEN + call malloc (buf, buflen, TY_STRUCT) + + # Read the full datafile header area into BUF. + repeat { + # Get the raw data. + nbytes = buflen * (SZ_STRUCT * SZB_CHAR) + call zardbf (chan, Memi[buf], nbytes, 1) + call zawtbf (chan, status) + if (status == ERR) + call syserrs (SYS_FMRERR, FM_DFNAME(fm)) + + # Extract the datafile header struct. + call miiupk32 (Memi[buf], Memi[dh], LEN_DHSTRUCT, TY_STRUCT) + if (DH_MAGIC(dh) != FMIO_MAGIC) + call syserrs (SYS_FMBADMAGIC, FM_DFNAME(fm)) + + # Repeat if the full header was not read. + if (DH_DATASTART(dh)-1 > nbytes) { + buflen = DH_DATASTART(dh)-1 / (SZ_STRUCT * SZB_CHAR) + call realloc (buf, buflen, TY_STRUCT) + } else if (status < DH_DATASTART(dh)-1) { + call syserrs (SYS_FMTRUNC, FM_DFNAME(fm)) + } else + break + } + + # Compute region of file in buffer. + b_off1 = 1 + b_off2 = b_off1 + buflen * SZ_STRUCT * SZB_CHAR + + # Copy general header fields. + FM_DFVERSION(fm) = DH_DFVERSION(dh) + FM_SZBPAGE(fm) = DH_SZBPAGE(dh) + FM_NLFILES(fm) = DH_NLFILES(dh) + FM_DATASTART(fm) = DH_DATASTART(dh) + FM_LSYNCTIME(fm) = clktime(0) + FM_DHMODIFIED(fm) = NO + + # Initialize buffer sizes. + szbpage = FM_SZBPAGE(fm) + call fmio_setbuf (fm) + if (FM_SZBPAGE(fm) != szbpage) + call syserrs (SYS_FMBLKCHSZ, FM_DFNAME(fm)) + + # Initialize the file table. + call calloc (ft, (FM_NLFILES(fm) + 1) * LEN_FTE, TY_STRUCT) + FM_FTOFF(fm) = DH_FTOFF(dh) + FM_FTLASTNF(fm) = DH_FTLASTNF(dh) + FM_FTABLE(fm) = ft + + ip = buf + FM_FTOFF(fm) - 1 + call miiupk32 (Memi[ip], Memi[ip], + (FM_NLFILES(fm) + 1) * LEN_FTEX, TY_INT) + + do i = 0, FM_NLFILES(fm) { + op = ft + i * LEN_FTE + LF_FSIZE(op) = FT_FSIZE(ip) + LF_FLAGS(op) = FT_FLAGS(ip) + ip = ip + LEN_FTEX + } + + # Read the page table index. + FM_PTIOFF(fm) = DH_PTIOFF(dh) + FM_PTILEN(fm) = DH_PTILEN(dh) + FM_PTINPTI(fm) = DH_PTINPTI(dh) + + ip = buf + FM_PTIOFF(fm) - 1 + call malloc (pti, FM_PTILEN(fm), TY_INT) + call miiupk32 (Memi[ip], Memi[pti], FM_PTILEN(fm), TY_INT) + FM_PTINDEX(fm) = pti + + # Now read the page table itself, stored in the data pages. + FM_PTLEN(fm) = DH_PTLEN(dh) + FM_PTNPTE(fm) = DH_PTNPTE(dh) + FM_PTLUPTE(fm) = DH_PTNPTE(dh) + + call malloc (pt, FM_PTLEN(fm), TY_SHORT) + FM_PTABLE(fm) = pt + + maxpages = FM_MAXBUFSIZE(fm) / FM_SZBPAGE(fm) + if (maxpages <= 0) + maxpages = DEF_BIGBUFNP + + op = pt + npti = FM_PTINPTI(fm) + for (p1=1; p1 <= npti; p1=p2) { + # Get a contiguous range of page table pages. + d1 = Memi[pti+p1-1] + for (p2=p1+1; p2 <= npti; p2=p2+1) { + d2 = Memi[pti+p2-1] + if (d2-d1 != p2-p1 || p2-p1 >= maxpages) + break + } + + # Read in the pages. + nbytes = (p2 - p1) * FM_SZBPAGE(fm) + nwords = nbytes / (SZB_CHAR * SZ_SHORT) + offset = (d1-1) * FM_SZBPAGE(fm) + FM_DATASTART(fm) + + # Check to see if data is in BUF before reading datafile. + if (offset >= b_off1 && (offset+nbytes) <= b_off2) + call bytmov (Memi[buf], offset, Mems[op], 1, nbytes) + else { + call zardbf (chan, Mems[op], nbytes, offset) + call zawtbf (chan, status) + if (status < nbytes) + call syserrs (SYS_FMTRUNC, FM_DFNAME(fm)) + } + + op = op + nwords + } + + # Swap the data. + if (BYTE_SWAP2 == YES) + call bswap2 (Mems[pt], 1, Mems[pt], 1, npti * FM_SZBPAGE(fm)) + + call mfree (buf, TY_STRUCT) + call sfree (sp) +end diff --git a/sys/fmio/fmiosbuf.x b/sys/fmio/fmiosbuf.x new file mode 100644 index 00000000..358d4136 --- /dev/null +++ b/sys/fmio/fmiosbuf.x @@ -0,0 +1,56 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <config.h> +include <fio.h> +include "fmio.h" + +# FMIO_SETBUF -- Set the various buffer size parameters, making sure that the +# physical constraints are met, i.e., that the buffers are an integral +# multiple of the device block size, and do not exceed the maximum transfer +# size for the device. + +procedure fmio_setbuf (fm) + +pointer fm #I FMIO descriptor + +int devblksize, devmaxsize +int optbufsize, maxbufsize, szbpage + +begin + # Get the device parameters for the device on which datafile resides. + call zsttbf (FM_CHAN(fm), FSTT_BLKSIZE, devblksize) + call zsttbf (FM_CHAN(fm), FSTT_MAXBUFSIZE, devmaxsize) + + # Make sure the page size is an integral multiple of the block size. + szbpage = (FM_SZBPAGE(fm) + devblksize-1) / devblksize * devblksize + + # Set the optimum (default) file buffer size. + optbufsize = FM_OPTBUFSIZE(fm) + if (optbufsize <= 0) { + if (DEF_OPTBUFNP <= 0) + call zsttbf (FM_CHAN(fm), FSTT_OPTBUFSIZE, optbufsize) + else + optbufsize = DEF_OPTBUFNP * szbpage + } + + # Set the maximum file buffer size. + maxbufsize = FM_MAXBUFSIZE(fm) + if (maxbufsize <= 0) { + if (DEF_MAXBUFNP > 0) + maxbufsize = DEF_MAXBUFNP * szbpage + else + maxbufsize = devmaxsize + } + + # Apply constraints and store values. + if (devmaxsize > 0) + maxbufsize = min (maxbufsize, devmaxsize) + if (maxbufsize > 0) + FM_MAXBUFSIZE(fm) = max (szbpage, maxbufsize / szbpage * szbpage) + + FM_OPTBUFSIZE(fm) = max (szbpage, optbufsize / szbpage * szbpage) + FM_DEVBLKSIZE(fm) = devblksize + FM_SZBPAGE(fm) = szbpage + + FM_DHMODIFIED(fm) = YES +end diff --git a/sys/fmio/fmiotick.x b/sys/fmio/fmiotick.x new file mode 100644 index 00000000..c29e035b --- /dev/null +++ b/sys/fmio/fmiotick.x @@ -0,0 +1,17 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmio.h" + +# FMIO_TICK -- Check the clock and do things that need to get done +# periodically, like sync the datafile. + +procedure fmio_tick (fm) + +pointer fm #I FMIO descriptor + +long clktime() + +begin + if (clktime(FM_LSYNCTIME(fm)) > SYNC_INTERVAL) + call fm_sync (fm) +end diff --git a/sys/fmio/fmlfard.x b/sys/fmio/fmlfard.x new file mode 100644 index 00000000..2e463f72 --- /dev/null +++ b/sys/fmio/fmlfard.x @@ -0,0 +1,29 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <mach.h> +include "fmio.h" + +# FM_LFAREAD -- Asynchronous blocked read from an lfile. The differences +# between text and binary lfiles are not visible above this level; both +# appear as binary files to FIO. + +procedure fm_lfaread (lf, buf, maxbytes, offset) + +pointer lf #I lfile descriptor +char buf[ARB] #O output data buffer +int maxbytes #I max bytes to read +long offset #I lfile offset + +int status, nb + +begin + # If reading text data, unpack the text data in place. + if (and (LF_FLAGS(lf), LFF_TEXTFILE) != 0) { + nb = (maxbytes + SZB_CHAR-1) / SZB_CHAR + call fm_lfbinread (lf, buf, nb, (offset-1) / SZB_CHAR + 1) + call fm_lfbinwait (lf, status) + if (status > 0) + call chrupk (buf, 1, buf, 1, min(maxbytes,status)) + } else + call fm_lfbinread (lf, buf, maxbytes, offset) +end diff --git a/sys/fmio/fmlfawr.x b/sys/fmio/fmlfawr.x new file mode 100644 index 00000000..7d122dc0 --- /dev/null +++ b/sys/fmio/fmlfawr.x @@ -0,0 +1,35 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <mach.h> +include "fmio.h" + +# FM_LFAWRITE -- Asynchronous blocked write to an lfile. The differences +# between text and binary lfiles are not visible above this level; both +# appear as binary files to FIO. + +procedure fm_lfawrite (lf, buf, nbytes, offset) + +pointer lf #I lfile descriptor +char buf[ARB] #O input data buffer +int nbytes #I nbytes to write +long offset #I lfile offset + +int status, nb +pointer sp, pk_buf + +begin + if (and (LF_FLAGS(lf), LFF_TEXTFILE) == 0) + call fm_lfbinwrite (lf, buf, nbytes, offset) + else { + call smark (sp) + call salloc (pk_buf, nbytes / SZB_CHAR, TY_CHAR) + + # Wait for i/o to complete before freeing buffer! + nb = (nbytes + SZB_CHAR-1) / SZB_CHAR + call chrpak (buf, 1, Memc[pk_buf], 1, nb) + call fm_lfbinwrite (lf, Memc[pk_buf], nb, (offset-1)/SZB_CHAR+1) + call fm_lfbinwait (lf, status) + + call sfree (sp) + } +end diff --git a/sys/fmio/fmlfawt.x b/sys/fmio/fmlfawt.x new file mode 100644 index 00000000..1fa66581 --- /dev/null +++ b/sys/fmio/fmlfawt.x @@ -0,0 +1,18 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <mach.h> +include "fmio.h" + +# FM_LFAWAIT -- Wait for i/o on an lfile. + +procedure fm_lfawait (lf, status) + +pointer lf #I lfile descriptor +int status #O i/o status (nbytes transferred or ERR) + +begin + call fm_lfbinwait (lf, status) + if (and (LF_FLAGS(lf), LFF_TEXTFILE) != 0) + if (status > 0) + status = status * SZB_CHAR +end diff --git a/sys/fmio/fmlfbrd.x b/sys/fmio/fmlfbrd.x new file mode 100644 index 00000000..acead281 --- /dev/null +++ b/sys/fmio/fmlfbrd.x @@ -0,0 +1,89 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <knet.h> +include <mach.h> +include "fmio.h" + +# FM_LFBINREAD -- Asynchronous blocked binary read from an lfile. We deal +# only with binary data here; unpacking of text must be done at a higher +# level. The basic procedure is to convert the indicated lfile segment into +# a range of lfile pages, then use the pagemap for the lfile to map these onto +# physical datafile pages. I/O is done in chunks of contiguous pages until +# the requested amount of data has been transferred. + +procedure fm_lfbinread (lf, buf, maxbytes, offset) + +pointer lf #I lfile descriptor +char buf[ARB] #O output data buffer +int maxbytes #I max bytes to read +long offset #I lfile offset + +pointer fm, pm +int status, chan, nleft, szbpage +int l1,l2, p1,p2, d1,d2, op, nb, np + +begin + fm = LF_FM(lf) + pm = LF_PAGEMAP(lf) + + # Verify descriptor. + if (fm == NULL || pm == NULL) { + LF_STATUS(lf) = ERR + return + } else + LF_STATUS(lf) = 0 + + np = LF_NPAGES(lf) + chan = FM_CHAN(fm) + + # Check that the read is in bounds. + nleft = min (offset + maxbytes, LF_FSIZE(lf) + 1) - offset + if (nleft <= 0) + return # read at EOF + + # Map lfile offset,nbytes into a range of lfile pages. + # I/O transfers are required to be aligned on page boundaries. + # Note that less than full page may be transferred in a read. + + szbpage = FM_SZBPAGE(fm) + l1 = (offset - 1) / szbpage + 1 + l2 = l1 + ((nleft + szbpage-1) / szbpage) - 1 + + # Read the data from the physical datafile into the user buffer, + # mapping lfile pages to physical offsets and moving data in chunks + # of as many contiguous pages as possible. + + op = 1 + for (p1=l1; nleft > 0 && p1 <= l2; p1=p2) { + # Get a contiguous range of datafile pages. + d1 = Memi[pm+p1-1] + for (p2=p1+1; p2 <= l2; p2=p2+1) { + d2 = Memi[pm+p2-1] + if (d2 - d1 != p2 - p1) + break + } + + # Read in the file segment. + nb = min (nleft, (p2 - p1) * szbpage) + call zardbf (chan, buf[op], nb, (d1-1)*szbpage + FM_DATASTART(fm)) + LF_FLAGS(lf) = or (LF_FLAGS(lf), LFF_IOINPROGRESS) + LF_LTSIZE(lf) = nb + + # Bump the i/o counters. + op = op + nb / SZB_CHAR + nleft = nleft - nb + + # If we didn't read all the data, wait until the read completes. + if (nleft > 0) { + call zawtbf (chan, status) + LF_FLAGS(lf) = and (LF_FLAGS(lf), not(LFF_IOINPROGRESS)) + if (status == ERR) { + LF_STATUS(lf) = ERR + return + } else if (status == 0) { + break + } else + LF_STATUS(lf) = LF_STATUS(lf) + status + } + } +end diff --git a/sys/fmio/fmlfbwr.x b/sys/fmio/fmlfbwr.x new file mode 100644 index 00000000..f16b71d1 --- /dev/null +++ b/sys/fmio/fmlfbwr.x @@ -0,0 +1,109 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <knet.h> +include <mach.h> +include "fmio.h" + +# FM_LFBINWRITE -- Asynchronous blocked binary write to an lfile. We deal +# only with binary data here; unpacking of text must be done at a higher +# level. The basic procedure is to convert the indicated lfile segment into +# a range of lfile pages, then use the pagemap for the lfile to map these onto +# physical datafile pages. I/O is done in chunks of contiguous pages until +# the requested amount of data has been transferred. When writing at or +# beyond EOF, new pages are automatically allocated upon demand. + +procedure fm_lfbinwrite (lf, buf, nbytes, offset) + +pointer lf #I lfile descriptor +char buf[ARB] #I input data buffer +int nbytes #I nbytes to write +long offset #I lfile offset + +pointer fm, pm +int status, chan, nleft, szbpage +int lfile, l1,l2, p1,p2, d1,d2, ip, nb, nt +int fmio_extend() + +begin + fm = LF_FM(lf) + pm = LF_PAGEMAP(lf) + + # Verify descriptor. + if (fm == NULL || pm == NULL) { + LF_STATUS(lf) = ERR + return + } else + LF_STATUS(lf) = 0 + + chan = FM_CHAN(fm) + szbpage = FM_SZBPAGE(fm) + lfile = (lf - FM_FTABLE(fm)) / LEN_FTE + nleft = nbytes + + # Extend the pagemap? + while (offset + nbytes > LF_NPAGES(lf)*szbpage + 1) + if (fmio_extend (fm, lfile, 1) == ERR) { + LF_STATUS(lf) = ERR + return + } else + pm = LF_PAGEMAP(lf) + + # Map lfile offset,nbytes into a range of lfile pages. + # I/O transfers are required to be aligned on page boundaries. + + l1 = (offset - 1) / szbpage + 1 + l2 = l1 + ((nleft + szbpage-1) / szbpage) - 1 + + # Write the data from the user buffer to the physical datafile, + # mapping lfile pages to physical offsets and moving data in chunks + # of as many contiguous pages as possible. + + ip = 1 + for (p1=l1; nleft > 0 && p1 <= l2; p1=p2) { + # Get a contiguous range of datafile pages. + d1 = Memi[pm+p1-1] + for (p2=p1+1; p2 <= l2; p2=p2+1) { + d2 = Memi[pm+p2-1] + if (d2 - d1 != p2 - p1) + break + } + + # Compute the logical transfer size NB, and the amount of data + # to be physically written NT. The latter is always an integral + # number of datafile pages in size. NOTE that this requires that + # the user buffer be an integral multiple of the page size, to + # prevent referencing off the end of the buffer. + + nb = min (nleft, (p2 - p1) * szbpage) + nt = (nb + szbpage-1) / szbpage * szbpage + LF_LTSIZE(lf) = nb + + # Write the file segment. + call zawrbf (chan, buf[ip], nt, (d1-1)*szbpage + FM_DATASTART(fm)) + LF_FLAGS(lf) = or (LF_FLAGS(lf), LFF_IOINPROGRESS) + + # Bump the i/o counters. + ip = ip + nb / SZB_CHAR + nleft = nleft - nb + + # If we didn't write all the data, wait until the write completes. + if (nleft > 0) { + call zawtbf (chan, status) + LF_FLAGS(lf) = and (LF_FLAGS(lf), not(LFF_IOINPROGRESS)) + if (status == ERR) { + LF_STATUS(lf) = ERR + return + } else if (status == 0) { + break + } else + LF_STATUS(lf) = LF_STATUS(lf) + min(LF_LTSIZE(lf),status) + } + } + + # Update the lfile size counter. + nb = offset + nbytes - 1 + if (nb > LF_FSIZE(lf)) { + LF_FSIZE(lf) = nb + FM_DHMODIFIED(fm) = YES + } +end diff --git a/sys/fmio/fmlfbwt.x b/sys/fmio/fmlfbwt.x new file mode 100644 index 00000000..2310c281 --- /dev/null +++ b/sys/fmio/fmlfbwt.x @@ -0,0 +1,32 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <knet.h> +include "fmio.h" + +# FM_LFBINWAIT -- Wait for i/o on a binary lfile. + +procedure fm_lfbinwait (lf, status) + +pointer lf #I lfile descriptor +int status #O i/o status (nbytes transferred or ERR) + +pointer fm +int chan + +begin + fm = LF_FM(lf) + chan = FM_CHAN(fm) + + # Wait for i/o and increment byte count. + if (and (LF_FLAGS(lf), LFF_IOINPROGRESS) != 0) { + call zawtbf (chan, status) + if (status >= 0) + LF_STATUS(lf) = LF_STATUS(lf) + min(LF_LTSIZE(lf),status) + else + LF_STATUS(lf) = ERR + LF_FLAGS(lf) = and (LF_FLAGS(lf), not(LFF_IOINPROGRESS)) + } + + call fmio_tick (fm) + status = LF_STATUS(lf) +end diff --git a/sys/fmio/fmlfcls.x b/sys/fmio/fmlfcls.x new file mode 100644 index 00000000..3084373d --- /dev/null +++ b/sys/fmio/fmlfcls.x @@ -0,0 +1,27 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmio.h" + +# FM_LFCLOSE -- Close an lfile descriptor. There isn't much for us to do, +# since the physical datafile remains open, and opening an lfile does not +# allocate a descriptor. + +procedure fm_lfclose (lf, status) + +pointer lf #I lfile descriptor +int status #O i/o status (nbytes transferred or ERR) + +pointer fm + +begin + fm = LF_FM(lf) + + if (fm == NULL) + status = ERR + else if (FM_MAGIC(fm) != FMIO_MAGIC) + status = ERR + else + status = OK + + call fmio_tick (fm) +end diff --git a/sys/fmio/fmlfcopy.x b/sys/fmio/fmlfcopy.x new file mode 100644 index 00000000..db937a7b --- /dev/null +++ b/sys/fmio/fmlfcopy.x @@ -0,0 +1,118 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <mach.h> +include "fmio.h" + +# FM_LFCOPY -- Copy an lfile, either to another lfile within the same +# datafile, or to another datafile. + +procedure fm_lfcopy (old, o_lfile, new, n_lfile) + +pointer old, new #I FMIO descriptors of source and destination +int o_lfile, n_lfile #I lfile numbers of source and dest. lfiles + +long offset +int n_szbpage +pointer sp, o_ft, n_ft, o_lf, n_lf, o_lfname, n_lfname, buf +int maxpages, maxbytes, nbytes, type, nleft, status + +errchk fmio_errchk, fmio_bind, syserrs +define olderr_ 91 +define newerr_ 92 + +begin + # Verify input. + if (o_lfile < 0 || o_lfile > FM_NLFILES(old)) + goto olderr_ + if (n_lfile < 0 || n_lfile > FM_NLFILES(new)) + goto newerr_ + + # Compute some useful values. + o_ft = FM_FTABLE(old) + n_ft = FM_FTABLE(new) + n_szbpage = FM_SZBPAGE(new) + maxpages = FM_MAXBUFSIZE(new) / n_szbpage + if (maxpages <= 0) + maxpages = DEF_BIGBUFNP + maxbytes = n_szbpage * maxpages + + # Copy the lfile. + o_lf = o_ft + o_lfile * LEN_FTE + n_lf = n_ft + n_lfile * LEN_FTE + + # Skip empty or deleted lfiles. + LF_FLAGS(n_lf) = and (LF_FLAGS(o_lf), not(LFF_DELETED)) + if (LF_FSIZE(o_lf) <= 0 || and(LF_FLAGS(o_lf),LFF_DELETED) != 0) + return + + # Get lfile type. + type = BINARY_FILE + if (and(LF_FLAGS(o_lf),LFF_TEXTFILE) != 0) + type = TEXT_FILE + + # Allocate buffers. + call smark (sp) + call salloc (o_lfname, SZ_FNAME, TY_CHAR) + call salloc (n_lfname, SZ_FNAME, TY_CHAR) + call salloc (buf, maxbytes / SZB_CHAR, TY_CHAR) + + # Open old lfile. + call fm_lfname (old, o_lfile, type, Memc[o_lfname], SZ_FNAME) + call strpak (Memc[o_lfname], Memc[o_lfname], SZ_FNAME) + call fm_lfopen (Memc[o_lfname], READ_ONLY, o_lf) + if (o_lf == ERR) + goto olderr_ + + # Open new lfile. + call fm_lfname (new, n_lfile, type, Memc[n_lfname], SZ_FNAME) + call strpak (Memc[n_lfname], Memc[n_lfname], SZ_FNAME) + call fm_lfopen (Memc[n_lfname], NEW_FILE, n_lf) + if (n_lf == ERR) + goto newerr_ + + # Copy the lfile data (as a binary file to avoid needless + # character unpack/pack). + + nleft = LF_FSIZE(o_lf) + for (offset=1; nleft > 0; offset=offset+nbytes) { + # Read a block of data. + call fm_lfbinread (o_lf, Memc[buf], maxbytes, offset) + call fm_lfbinwait (o_lf, nbytes) + if (nbytes == ERR) + goto olderr_ + else if (nbytes == 0) + break + + # Write to the new datafile. + call fm_lfbinwait (n_lf, status) + if (status == ERR) + goto newerr_ + call fm_lfbinwrite (n_lf, Memc[buf], nbytes, offset) + nleft = nleft - nbytes + } + + # Wait for the last write to terminate. + call fm_lfbinwait (n_lf, status) + if (status == ERR) + goto newerr_ + + # Close the lfiles. + call fm_lfclose (o_lf, status) + if (status == ERR) + goto olderr_ + call fm_lfclose (n_lf, status) + if (status == ERR) + goto newerr_ + + call fmio_errchk (old) + call fmio_errchk (new) + + call sfree (sp) + return + +olderr_ + call syserrs (SYS_FMLFCOPY, FM_DFNAME(old)) +newerr_ + call syserrs (SYS_FMLFCOPY, FM_DFNAME(new)) +end diff --git a/sys/fmio/fmlfdel.x b/sys/fmio/fmlfdel.x new file mode 100644 index 00000000..001de927 --- /dev/null +++ b/sys/fmio/fmlfdel.x @@ -0,0 +1,29 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include "fmio.h" + +# FM_LFDELETE -- Delete an lfile. + +procedure fm_lfdelete (fm, lfile) + +pointer fm #I FMIO descriptor +int lfile #I lfile number + +pointer lf +errchk syserrs, fmio_bind, fmio_errchk + +begin + call fmio_bind (fm) + call fmio_errchk (fm) + + # Verify input. + if (lfile < 0 || lfile > FM_NLFILES(fm)) + call syserrs (SYS_FMLFNOOB, FM_DFNAME(fm)) + + lf = FM_FTABLE(fm) + lfile * LEN_FTE + LF_FLAGS(lf) = or (LF_FLAGS(lf), LFF_DELETED) + + FM_DHMODIFIED(fm) = YES + call fmio_tick (fm) +end diff --git a/sys/fmio/fmlfname.x b/sys/fmio/fmlfname.x new file mode 100644 index 00000000..1d0b92e7 --- /dev/null +++ b/sys/fmio/fmlfname.x @@ -0,0 +1,45 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <fmset.h> +include "fmio.h" + +# FM_LFNAME -- Encode the pseudo-filename for an lfile. This is necessary to +# pass all lfile info through FIO, when opening a file descriptor on an lfile. +# +# The filename syntax is "Tddd.fff" where +# +# T is 'B' or 'T' for text or binary +# ddd is the encoded descriptor pointer +# fff is the encoded lfile number + +procedure fm_lfname (fm, lfile, type, lfname, maxch) + +pointer fm #I FMIO descriptor +int lfile #I lfile number +int type #I file type, text or binary +char lfname[maxch] #O encoded lfile filename +int maxch #I max chars out + +int op +int itoc() +errchk fmio_bind, fmio_errchk + +begin + call fmio_bind (fm) + call fmio_errchk (fm) + if (maxch <= 0) + return + + op = 1 + if (type == TEXT_FILE) + lfname[op] = 'T' + else + lfname[op] = 'B' + op = min (maxch, op + 1) + + op = min (maxch, op + itoc (fm, lfname[op], maxch-op+1)) + lfname[op] = '.' + op = min (maxch, op + 1) + op = min (maxch, op + itoc (lfile, lfname[op], maxch-op+1)) + lfname[op] = EOS +end diff --git a/sys/fmio/fmlfopen.x b/sys/fmio/fmlfopen.x new file mode 100644 index 00000000..a267834a --- /dev/null +++ b/sys/fmio/fmlfopen.x @@ -0,0 +1,89 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmio.h" + +# FM_LFOPEN -- Open an lfile. This routine is designed to be called with a +# pseudo-filename identifying the FMIO datafile, lfile therein, and file type. +# This is necessary to conform to the binary file driver interface standard. + +procedure fm_lfopen (pk_lfname, mode, chan) + +char pk_lfname[ARB] #I encoded lfile specification (packed char) +int mode #I file access mode +int chan #O i/o channel assigned (descriptor) + +int flags, lfile, type, np, i +pointer sp, lfname, fm, lf, pt, pm +int kmalloc(), krealloc(), fm_lfparse() +define err_ 91 + +begin + call smark (sp) + call salloc (lfname, SZ_FNAME, TY_CHAR) + + flags = 0 + + # Parse the file spec. + call strupk (pk_lfname, Memc[lfname], SZ_FNAME) + if (fm_lfparse (Memc[lfname], fm, lfile, type) == ERR) + goto err_ + else if (type == TEXT_FILE) + flags = flags + LFF_TEXTFILE + + # Verify input. + if (FM_MAGIC(fm) != FMIO_MAGIC) + goto err_ + else if (lfile < 0 || lfile > FM_NLFILES(fm)) + goto err_ + else if (lfile == PT_LFILE && mode != READ_ONLY) + goto err_ # protect page table! + + lf = FM_FTABLE(fm) + lfile * LEN_FTE + + # Activate the descriptor? + if (LF_PAGEMAP(lf) == NULL) { + LF_PMLEN(lf) = DEF_PMLEN + if (kmalloc (LF_PAGEMAP(lf), DEF_PMLEN, TY_INT) == ERR) + goto err_ + + pm = LF_PAGEMAP(lf) + pt = FM_PTABLE(fm) + np = 0 + + # Determine the lfile pages from the global page table. + do i = 1, FM_PTNPTE(fm) + if (Mems[pt+i-1] == lfile) { + np = np + 1 + if (np > LF_PMLEN(lf)) { + LF_PMLEN(lf) = (np+INC_PMLEN-1) / INC_PMLEN * INC_PMLEN + if (krealloc (pm, LF_PMLEN(lf), TY_INT) == ERR) + goto err_ + LF_PAGEMAP(lf) = pm + } + Memi[pm+np-1] = i + } + + LF_NPAGES(lf) = np + } + + # Mode dependent processing. + if (mode == NEW_FILE || and (LF_FLAGS(lf), LFF_DELETED) != 0) { + LF_FSIZE(lf) = 0 + LF_FLAGS(lf) = flags + } + + LF_FM(lf) = fm + LF_STATUS(lf) = 0 + LF_FLAGS(lf) = or (LFF_ALLOCATED, + and (LF_FLAGS(lf), not(LFF_IOINPROGRESS))) + + FM_DHMODIFIED(fm) = YES + + chan = lf + call fmio_tick (fm) + call sfree (sp) + return +err_ + chan = ERR + call sfree (sp) +end diff --git a/sys/fmio/fmlfparse.x b/sys/fmio/fmlfparse.x new file mode 100644 index 00000000..63f43a84 --- /dev/null +++ b/sys/fmio/fmlfparse.x @@ -0,0 +1,45 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <fmset.h> + +# FM_LFPARSE -- Parse an encoded lfile filename. +# The filename syntax is "Tddd.fff" where +# +# T is 'B' or 'T' for text or binary +# ddd is the encoded descriptor pointer +# fff is the encoded lfile number + +int procedure fm_lfparse (lfname, fm, lfile, type) + +char lfname[ARB] #I encoded lfile filename +pointer fm #O FMIO descriptor +int lfile #O lfile number +int type #O lfile file type (text or binary) + +int ip +int ctoi() + +begin + # Determine file type. + if (lfname[1] == 'T') + type = TEXT_FILE + else + type = BINARY_FILE + + # Get FMIO descriptor. + ip = 2 + if (ctoi (lfname, ip, fm) <= 0) + return (ERR) + + # Skip . delimiter. + if (lfname[ip] == '.') + ip = ip + 1 + else + return (ERR) + + # Get lfile number. + if (ctoi (lfname, ip, lfile) <= 0) + return (ERR) + + return (OK) +end diff --git a/sys/fmio/fmlfstat.h b/sys/fmio/fmlfstat.h new file mode 100644 index 00000000..14df119b --- /dev/null +++ b/sys/fmio/fmlfstat.h @@ -0,0 +1,10 @@ +# FMLFSTAT.H -- Lfile status structure definitions. + +# Lfstat structure. +define LEN_LFSTAT 2 # struct size +define LFU_SIZE $1[1] # lfile size, bytes +define LFU_FLAGS $1[2] # lfile flag bits + +# Flag bits. +define LFB_DELETED 1B # delete bit +define LFB_TEXTFILE 2B # file contains packed text diff --git a/sys/fmio/fmlfstat.x b/sys/fmio/fmlfstat.x new file mode 100644 index 00000000..a5829de0 --- /dev/null +++ b/sys/fmio/fmlfstat.x @@ -0,0 +1,31 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <fmlfstat.h> +include "fmio.h" + +# FMLFSTAT.H -- Query the attributes of an lfile. + +int procedure fm_lfstat (fm, lfile, statbuf) + +pointer fm #I FMIO descriptor +int lfile #I lfile number +int statbuf[ARB] #O receives status + +pointer lf +errchk fmio_bind, fmio_errchk + +begin + call fmio_bind (fm) + call fmio_errchk (fm) + + # Verify input. + if (lfile < 0 || lfile > FM_NLFILES(fm)) + return (ERR) + + # Copy out the lfile status. + lf = FM_FTABLE(fm) + lfile * LEN_FTE + LFU_SIZE(statbuf) = LF_FSIZE(lf) + LFU_FLAGS(statbuf) = LF_FLAGS(lf) + + return (OK) +end diff --git a/sys/fmio/fmlfstt.x b/sys/fmio/fmlfstt.x new file mode 100644 index 00000000..951c44c2 --- /dev/null +++ b/sys/fmio/fmlfstt.x @@ -0,0 +1,38 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <config.h> +include <mach.h> +include <fio.h> +include "fmio.h" + +# FM_LFSTATI -- Stat an lfile. + +procedure fm_lfstati (lf, param, lvalue) + +pointer lf #I lfile descriptor +int param #I parameter code +long lvalue #O parameter value + +pointer fm +int chan + +begin + fm = LF_FM(lf) + chan = FM_CHAN(fm) + + # Only the file size differs for each lfile. + switch (param) { + case FSTT_FILSIZE: + lvalue = LF_FSIZE(lf) + case FSTT_BLKSIZE: + lvalue = FM_SZBPAGE(fm) + case FSTT_OPTBUFSIZE: + lvalue = FM_OPTBUFSIZE(fm) + case FSTT_MAXBUFSIZE: + lvalue = FM_MAXBUFSIZE(fm) + } + + # For text lfiles, things appear to be SZB_CHAR larger. + if (and (LF_FLAGS(lf), LFF_TEXTFILE) != 0) + lvalue = lvalue * SZB_CHAR +end diff --git a/sys/fmio/fmlfundel.x b/sys/fmio/fmlfundel.x new file mode 100644 index 00000000..98c822ee --- /dev/null +++ b/sys/fmio/fmlfundel.x @@ -0,0 +1,28 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include "fmio.h" + +# FM_LFUNDELETE -- Undelete an lfile. + +procedure fm_lfundelete (fm, lfile) + +pointer fm #I FMIO descriptor +int lfile #I lfile number + +pointer lf +errchk syserrs, fmio_bind, fmio_errchk + +begin + call fmio_bind (fm) + call fmio_errchk (fm) + + # Verify input. + if (lfile < 0 || lfile > FM_NLFILES(fm)) + call syserrs (SYS_FMLFNOOB, FM_DFNAME(fm)) + + lf = FM_FTABLE(fm) + lfile * LEN_FTE + LF_FLAGS(lf) = and (LF_FLAGS(lf), not(LFF_DELETED)) + FM_DHMODIFIED(fm) = YES + call fmio_tick (fm) +end diff --git a/sys/fmio/fmnextlf.x b/sys/fmio/fmnextlf.x new file mode 100644 index 00000000..0747a799 --- /dev/null +++ b/sys/fmio/fmnextlf.x @@ -0,0 +1,48 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include "fmio.h" + +# FM_NEXTLFILE -- Return the next available (empty) lfile. An error action +# is taken if all lfiles are currently in use. Deleted lfiles will be reused +# if no unused lfiles are found. + +int procedure fm_nextlfile (fm) + +pointer fm #I FMIO descriptor + +pointer ft, lf +int nlfiles, flags, fn, i +errchk syserrs, fmio_bind, fmio_errchk + +begin + call fmio_bind (fm) + call fmio_errchk (fm) + + fn = FM_FTLASTNF(fm) + ft = FM_FTABLE(fm) + nlfiles = FM_NLFILES(fm) + + # Travel once around the file table and abort if all entries are used. + # New files are normally returned in sequence. Deleted files can be + # reused. + + do i = 1, nlfiles { + fn = fn + 1 + if (fn > nlfiles) + fn = 1 + lf = ft + fn * LEN_FTE + flags = LF_FLAGS(lf) + if (and(flags,LFF_ALLOCATED) == 0 || and(flags,LFF_DELETED) != 0) + break + } + + if (i > nlfiles) + call syserrs (SYS_FMOOF, FM_DFNAME(fm)) + + FM_FTLASTNF(fm) = fn + FM_DHMODIFIED(fm) = YES + call fmio_tick (fm) + + return (fn) +end diff --git a/sys/fmio/fmopen.x b/sys/fmio/fmopen.x new file mode 100644 index 00000000..8a8657ba --- /dev/null +++ b/sys/fmio/fmopen.x @@ -0,0 +1,67 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <error.h> +include <knet.h> +include "fmio.h" + +# FM_OPEN -- Open or create a FMIO datafile. Since this is a low level +# interface we do not want to impose a choice of a file extension on the +# datafile, so if a file extension is desired it must be supplied. + +pointer procedure fm_open (fname, mode) + +char fname[ARB] #I datafile filename +int mode #I file access mode + +int chan, n +pointer sp, osfn, fn, fm +errchk syserrs, calloc, fclobber +int nowhite() + +begin + call smark (sp) + call salloc (fn, SZ_PATHNAME, TY_CHAR) + call salloc (osfn, SZ_PATHNAME, TY_CHAR) + + n = nowhite (fname, Memc[fn], SZ_PATHNAME) + + # Take care to not clobber an existing file. + if (mode == NEW_FILE) + call fclobber (Memc[fn]) + + # Open the datafile. + call fmapfn (Memc[fn], Memc[osfn], SZ_PATHNAME) + call zopnbf (Memc[osfn], mode, chan) + if (chan == ERR) + call syserrs (SYS_FMOPEN, Memc[fn]) + + # Allocate the FMIO descriptor. + call calloc (fm, LEN_FMDES, TY_STRUCT) + + FM_MODE(fm) = mode + FM_CHAN(fm) = chan + FM_MAGIC(fm) = FMIO_MAGIC + FM_SZFCACHE(fm) = DEF_FCACHESIZE + call strcpy (Memc[fn], FM_DFNAME(fm), SZ_DFNAME) + + if (mode == NEW_FILE) { + # Wait until first i/o operation to finish initialization. + FM_DFVERSION(fm) = FMIO_VERSION + FM_SZBPAGE(fm) = DEF_PAGESIZE + FM_NLFILES(fm) = DEF_MAXLFILES + FM_PTILEN(fm) = DEF_MAXPTPAGES + FM_ACTIVE(fm) = NO + + } else { + # Open an existing datafile. + iferr (call fmio_readheader(fm)) { + call mfree (fm, TY_STRUCT) + call erract (EA_ERROR) + } else + FM_ACTIVE(fm) = YES + } + + call sfree (sp) + return (fm) +end diff --git a/sys/fmio/fmrebuild.x b/sys/fmio/fmrebuild.x new file mode 100644 index 00000000..9eb55656 --- /dev/null +++ b/sys/fmio/fmrebuild.x @@ -0,0 +1,26 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +# FM_REBUILD -- Rebuild a datafile. This has no affect on the logical content +# of a datafile, but is desirable for efficiency reasons to eliminate waste +# space (e.g., from deleted lfiles), and render the file structures logically +# contiguous within the file, increasing i/o access efficiency. + +procedure fm_rebuild (dfname) + +char dfname[ARB] #I datafile name + +pointer sp, tfname +errchk fm_copy, fm_delete, fm_rename + +begin + call smark (sp) + call salloc (tfname, SZ_PATHNAME, TY_CHAR) + + # The copy operation rebuilds a datafile. + call mktemp (dfname, Memc[tfname], SZ_PATHNAME) + call fm_copy (dfname, Memc[tfname]) + call fm_delete (dfname) + call fm_rename (Memc[tfname], dfname) + + call sfree (sp) +end diff --git a/sys/fmio/fmrename.x b/sys/fmio/fmrename.x new file mode 100644 index 00000000..e5221536 --- /dev/null +++ b/sys/fmio/fmrename.x @@ -0,0 +1,11 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +# FM_RENAME -- Rename (or move) a datafile. + +procedure fm_rename (old, new) + +char old[ARB], new[ARB] #I old, new datafile names + +begin + call rename (old, new) +end diff --git a/sys/fmio/fmset.h b/sys/fmio/fmset.h new file mode 100644 index 00000000..a271cc27 --- /dev/null +++ b/sys/fmio/fmset.h @@ -0,0 +1,24 @@ +# FMSET.H -- User definitions for FMIO. + +# SET/STAT codes. +define FM_ACMODE 1 #RO datafile access mode +define FM_FCACHESIZE 2 #RW number of files in open file cache +define FM_MAXFBSIZE 3 #RW maximum lfile-FIO buffer size +define FM_MAXLFILES 4 #RW number of lfiles in datafile +define FM_MAXPTPAGES 5 #RW max page table pages (max filesize) +define FM_OPTFBSIZE 6 #RW default lfile-FIO buffer size +define FM_OSCHAN 7 #RO os channel of datafile +define FM_PAGESIZE 8 #RW datafile page size, bytes +define FM_VERSION 9 #RO FMIO version number of datafile + +# FM_DEBUG flags. +define FMD_HEADER 001B # general header parameters +define FMD_FTABLE 002B # summarize file table contents +define FMD_PTINDEX 004B # print page table index +define FMD_PTABLE 010B # print page table +define FMD_ALL 017B # print everything + +# FM_FCDEBUG flags. +define FCD_CACHE 001B # print current cache status +define FCD_LFSTATISTICS 002B # print statistics on lfile getfd's +define FCD_ALL 003B # print everything diff --git a/sys/fmio/fmseti.x b/sys/fmio/fmseti.x new file mode 100644 index 00000000..84cefc27 --- /dev/null +++ b/sys/fmio/fmseti.x @@ -0,0 +1,39 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmset.h" +include "fmio.h" + +# FM_SETI -- Set the value of an FMIO integer parameter. + +procedure fm_seti (fm, param, value) + +pointer fm #I FMIO descriptor +int param #I parameter code from <fmset.h> +int value #I new parameter value + +int szbpage + +begin + szbpage = FM_SZBPAGE(fm) + + switch (param) { + case FM_ACMODE: + ; # read-only + case FM_MAXLFILES: + FM_NLFILES(fm) = value + case FM_MAXPTPAGES: + FM_PTILEN(fm) = value + case FM_OSCHAN: + FM_CHAN(fm) = value + case FM_PAGESIZE: + FM_SZBPAGE(fm) = value + case FM_VERSION: + ; # read-only + case FM_OPTFBSIZE: + FM_OPTBUFSIZE(fm) = value + case FM_MAXFBSIZE: + FM_MAXBUFSIZE(fm) = value + case FM_FCACHESIZE: + FM_SZFCACHE(fm) = value + } +end diff --git a/sys/fmio/fmstati.x b/sys/fmio/fmstati.x new file mode 100644 index 00000000..82720ade --- /dev/null +++ b/sys/fmio/fmstati.x @@ -0,0 +1,36 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include "fmset.h" +include "fmio.h" + +# FM_STATI -- Query an FMIO integer parameter. + +int procedure fm_stati (fm, param) + +pointer fm #I FMIO descriptor +int param #I parameter code from <fmset.h> + +begin + switch (param) { + case FM_ACMODE: + return (FM_MODE(fm)) + case FM_MAXLFILES: + return (FM_NLFILES(fm)) + case FM_MAXPTPAGES: + return (FM_PTILEN(fm)) + case FM_OSCHAN: + return (FM_CHAN(fm)) + case FM_PAGESIZE: + return (FM_SZBPAGE(fm)) + case FM_VERSION: + return (FM_DFVERSION(fm)) + case FM_OPTFBSIZE: + return (FM_OPTBUFSIZE(fm)) + case FM_MAXFBSIZE: + return (FM_MAXBUFSIZE(fm)) + case FM_FCACHESIZE: + return (FM_SZFCACHE(fm)) + default: + return (ERR) + } +end diff --git a/sys/fmio/fmsync.x b/sys/fmio/fmsync.x new file mode 100644 index 00000000..7c7d8112 --- /dev/null +++ b/sys/fmio/fmsync.x @@ -0,0 +1,169 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <syserr.h> +include <mach.h> +include <knet.h> +include "fmio.h" + +define SZB_SHORT (SZB_CHAR*SZ_SHORT) + +# FM_SYNC -- Update any buffered portions of the datafile on disk which have +# been modified since the last update. + +procedure fm_sync (fm) + +pointer fm #I FMIO descriptor + +pointer sp, ip, op, dhbuf, pgbuf, pti, dh, ft, pt +int maxpages, nbytes, npti, p1, p2, d1, d2, dp, i +int szbpage, chan, buflen, status, npte_perpage +int fmio_extend() +long clktime() + +begin + if (FM_MODE(fm) == READ_ONLY) + return + + call smark (sp) + chan = FM_CHAN(fm) + szbpage = FM_SZBPAGE(fm) + npte_perpage = szbpage / SZB_SHORT + + call intr_disable() + + # Get more page table space (pages). During a normal file extend + # occurring while writing to a file, PTEs for *data* pages are added + # to the incore global page table and to the PT for the lfile. + # The additional pages needed to store the PT and PTI as they grow + # are however not allocated during i/o. We wait and do this at sync + # time to provide maximal separation of the data pages and PT pages, + # rendering both as contiguous as possible. + + # Check for page table index overflow (datafile too large). + npti = (FM_PTNPTE(fm) + npte_perpage-1) / npte_perpage + if (npti > FM_PTILEN(fm)) { + call fmio_posterr (fm, SYS_FMPTIOVFL, FM_DFNAME(fm)) + + # Truncate the page table to try to recover the datafile with + # some loss of data. + + npti = FM_PTILEN(fm) + FM_PTNPTE(fm) = npti * npte_perpage - (npti - FM_PTINPTI(fm)) + } + + # Allocate the page table pages. + for (p1=FM_PTINPTI(fm)+1; p1 <= npti; p1=p1+1) { + dp = fmio_extend (fm, PT_LFILE, 1) + if (dp != ERR) { + Memi[FM_PTINDEX(fm)+p1-1] = dp + FM_PTINPTI(fm) = p1 + FM_DHMODIFIED(fm) = YES + } else + call fmio_posterr (fm, SYS_FMPTIOVFL, FM_DFNAME(fm)) + } + + # Update the datafile header area. + if (FM_DHMODIFIED(fm) != NO) { + # Allocate a buffer to hold the encoded datafile header area. + buflen = (FM_DATASTART(fm) - 1) / (SZ_STRUCT * SZB_CHAR) + call salloc (dhbuf, buflen, TY_STRUCT) + + # Encode and output the datafile header. + call salloc (dh, LEN_DHSTRUCT, TY_STRUCT) + + DH_MAGIC(dh) = FMIO_MAGIC + DH_DFVERSION(dh) = FM_DFVERSION(fm) + DH_SZBPAGE(dh) = szbpage + DH_NLFILES(dh) = FM_NLFILES(fm) + DH_FTOFF(dh) = FM_FTOFF(fm) + DH_FTLASTNF(dh) = FM_FTLASTNF(fm) + DH_PTIOFF(dh) = FM_PTIOFF(fm) + DH_PTILEN(dh) = FM_PTILEN(fm) + DH_PTINPTI(dh) = FM_PTINPTI(fm) + DH_PTLEN(dh) = FM_PTLEN(fm) + DH_PTNPTE(dh) = FM_PTNPTE(fm) + DH_DATASTART(dh) = FM_DATASTART(fm) + + call miipak32 (Memi[dh], Memi[dhbuf], LEN_DHSTRUCT, TY_STRUCT) + + # Output the file table. + ft = FM_FTABLE(fm) + op = dhbuf + FM_FTOFF(fm) - 1 + do i = 0, FM_NLFILES(fm) { + ip = ft + i * LEN_FTE + FT_FSIZE(op) = LF_FSIZE(ip) + FT_FLAGS(op) = and (LFF_SAVE, LF_FLAGS(ip)) + op = op + LEN_FTEX + } + + op = dhbuf + FM_FTOFF(fm) - 1 + call miipak32 (Memi[op], Memi[op], + (FM_NLFILES(fm) + 1) * LEN_FTEX, TY_INT) + + # Output the page table index. + call miipak32 (Memi[FM_PTINDEX(fm)], Memi[dhbuf+FM_PTIOFF(fm)-1], + FM_PTILEN(fm), TY_INT) + + # Update the whole thing on disk. + call zawrbf (chan, Memi[dhbuf], FM_DATASTART(fm)-1, 1) + call zawtbf (chan, status) + if (status == ERR) + call fmio_posterr (fm, SYS_FMWRERR, FM_DFNAME(fm)) + + FM_DHMODIFIED(fm) = NO + } + + # Don't cache these pointers before calling fmio_extend! + pt = FM_PTABLE(fm) + pti = FM_PTINDEX(fm) + + # Update the page table itself, stored in the data pages. + if (FM_PTNPTE(fm) > FM_PTLUPTE(fm)) { + + # Determine the max transfer size. + maxpages = FM_MAXBUFSIZE(fm) / szbpage + if (maxpages <= 0) + maxpages = DEF_BIGBUFNP + + # Get an output temporary buffer if we have to swap on output. + if (BYTE_SWAP2 == YES) + call salloc (pgbuf, maxpages * szbpage / SZB_SHORT, TY_SHORT) + + # Determine the PT page containing the LUPTE+1. + p1 = FM_PTLUPTE(fm) / npte_perpage + 1 + + # Update that and all following PT pages. + npti = FM_PTINPTI(fm) + for (; p1 <= npti; p1=p2) { + # Get a contiguous range of page table pages. + d1 = Memi[pti+p1-1] + for (p2=p1+1; p2 <= npti; p2=p2+1) { + d2 = Memi[pti+p2-1] + if (d2-d1 != p2-p1 || p2-p1 >= maxpages) + break + } + + # Swap the data and set IP for the output transfer. + ip = pt + (p1 - 1) * npte_perpage + nbytes = (min(npti+1,p2) - p1) * szbpage + if (BYTE_SWAP2 == YES) { + call bswap2 (Mems[ip], 1, Mems[pgbuf], 1, nbytes) + ip = pgbuf + } + + # Update the pages. + call zawrbf (chan, Mems[ip], nbytes, + (d1-1) * szbpage + FM_DATASTART(fm)) + call zawtbf (chan, status) + if (status < nbytes) + call fmio_posterr (fm, SYS_FMWRERR, FM_DFNAME(fm)) + } + + FM_PTLUPTE(fm) = FM_PTNPTE(fm) + } + + FM_LSYNCTIME(fm) = clktime(0) + call intr_enable() + + call sfree (sp) +end diff --git a/sys/fmio/mkpkg b/sys/fmio/mkpkg new file mode 100644 index 00000000..0daf637a --- /dev/null +++ b/sys/fmio/mkpkg @@ -0,0 +1,52 @@ +# Make the FMIO (file manager i/o) library. + +$checkout libsys.a lib$ +$update libsys.a +$checkin libsys.a lib$ +$exit + +zzdebug: +zzdebug.e: + $omake zzdebug.x <error.h> <ctype.h> fmset.h + $link zzdebug.o + ; + +libsys.a: + fmaccess.x + fmclose.x fmio.h <error.h> <knet.h> + fmcopy.x fmset.h <error.h> + fmcopyo.x fmio.h <mach.h> + fmdebug.x fmio.h fmset.h + fmdelete.x + fmfcache.x fmio.h fmset.h <error.h> <fset.h> + fmfopen.x + fmiobind.x fmio.h <mach.h> + fmioerr.x fmio.h + fmioextnd.x fmio.h <mach.h> + fmiopost.x fmio.h + fmiorhdr.x fmio.h <knet.h> <mach.h> + fmiosbuf.x fmio.h <config.h> <fio.h> + fmiotick.x fmio.h + fmlfard.x fmio.h <mach.h> + fmlfawr.x fmio.h <mach.h> + fmlfawt.x fmio.h <mach.h> + fmlfbrd.x fmio.h <knet.h> <mach.h> + fmlfbwr.x fmio.h <knet.h> <mach.h> + fmlfbwt.x fmio.h <knet.h> + fmlfcls.x fmio.h + fmlfcopy.x fmio.h <mach.h> + fmlfdel.x fmio.h + fmlfname.x fmio.h <fmset.h> + fmlfopen.x fmio.h + fmlfparse.x <fmset.h> + fmlfstat.x fmio.h <fmlfstat.h> + fmlfstt.x fmio.h <config.h> <fio.h> <mach.h> + fmlfundel.x fmio.h + fmnextlf.x fmio.h + fmopen.x fmio.h <error.h> <knet.h> + fmrebuild.x + fmrename.x + fmseti.x fmio.h fmset.h + fmstati.x fmio.h fmset.h + fmsync.x fmio.h <knet.h> <mach.h> + ; diff --git a/sys/fmio/zzdebug.x b/sys/fmio/zzdebug.x new file mode 100644 index 00000000..1764d62e --- /dev/null +++ b/sys/fmio/zzdebug.x @@ -0,0 +1,303 @@ +# Copyright(c) 1986 Association of Universities for Research in Astronomy Inc. + +include <error.h> +include <ctype.h> +include <fmlfstat.h> +include <mach.h> +include "fmset.h" + +# ZZDEBUG.X -- Debug routines for the FMIO package. + +task create = t_create, + enter = t_enter, + extract = t_extract, + mkfile = t_mkfile, + type = t_type, + show = t_show, + copy = t_copy, + rebuild = t_rebuild, + fcache = t_fcache + + +# CREATE -- Create a new, empty datafile. + +procedure t_create() + +pointer fm +char datafile[SZ_FNAME] +int pagesize, nlfiles, maxptpages + +int clgeti() +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + pagesize = clgeti ("pagesize") + nlfiles = clgeti ("nlfiles") + maxptpages = clgeti ("maxptpages") + + fm = fm_open (datafile, NEW_FILE) + if (pagesize > 0) + call fm_seti (fm, FM_PAGESIZE, pagesize) + if (nlfiles > 0) + call fm_seti (fm, FM_MAXLFILES, nlfiles) + if (maxptpages > 0) + call fm_seti (fm, FM_MAXPTPAGES, maxptpages) + call fm_close (fm) +end + + +# ENTER -- Copy a regular disk file into an lfile. + +procedure t_enter() + +pointer fm +int lfile, type, fd, lf +char datafile[SZ_FNAME], fname[SZ_FNAME] +int clgeti(), open(), fm_fopen(), access() +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + call clgstr ("fname", fname, SZ_FNAME) + lfile = clgeti ("lfile") + + if (access (fname, 0, TEXT_FILE) == YES) + type = TEXT_FILE + else + type = BINARY_FILE + + fm = fm_open (datafile, READ_WRITE) + fd = open (fname, READ_ONLY, type) + lf = fm_fopen (fm, lfile, NEW_FILE, type) + call fcopyo (fd, lf) + + call close (lf) + call close (fd) + call fm_close (fm) +end + + +# EXTRACT -- Copy an lfile out into a disk file. + +procedure t_extract() + +pointer fm +int lfstat[LEN_LFSTAT] +int lfile, type, fd, lf +char datafile[SZ_FNAME], fname[SZ_FNAME] +int clgeti(), open(), fm_fopen(), fm_lfstat() +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + call clgstr ("fname", fname, SZ_FNAME) + lfile = clgeti ("lfile") + + fm = fm_open (datafile, READ_ONLY) + + if (fm_lfstat (fm, lfile, lfstat) == ERR) + call error (1, "cannot stat lfile") + else if (and (LFU_FLAGS(lfstat), LFB_TEXTFILE) != 0) + type = TEXT_FILE + else + type = BINARY_FILE + + lf = fm_fopen (fm, lfile, READ_ONLY, type) + fd = open (fname, NEW_FILE, type) + call fcopyo (lf, fd) + + call close (lf) + call close (fd) + call fm_close (fm) +end + + +# MKFILE -- Create a file of the given size (in kilobytes) containing all +# zero data. + +procedure t_mkfile() + +pointer fm +int lfile, lf, kb, i +char datafile[SZ_FNAME], buf[1024/SZB_CHAR] +int clgeti(), fm_fopen() +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + lfile = clgeti ("lfile") + kb = clgeti ("kb") + + fm = fm_open (datafile, READ_WRITE) + lf = fm_fopen (fm, lfile, NEW_FILE, BINARY_FILE) + + do i = 1, kb + iferr (call write (lf, buf, 1024/SZB_CHAR)) { + call erract (EA_WARN) + break + } + + call close (lf) + call fm_close (fm) +end + + +# TYPE -- Print the contents of an lfile on the standard output. + +procedure t_type() + +pointer fm +int lfile, fd +char datafile[SZ_FNAME] +int clgeti(), fm_fopen() +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + lfile = clgeti ("lfile") + + fm = fm_open (datafile, READ_WRITE) + fd = fm_fopen (fm, lfile, READ_ONLY, TEXT_FILE) + call fcopyo (fd, STDOUT) + + call close (fd) + call fm_close (fm) +end + + +# SHOW -- Print the datafile status. + +procedure t_show() + +pointer fm +char datafile[SZ_FNAME] +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + + fm = fm_open (datafile, READ_WRITE) + call fm_debug (fm, STDOUT, FMD_ALL) + call fm_close (fm) +end + + +# COPY -- Copy a datafile. + +procedure t_copy() + +char df_src[SZ_FNAME] +char df_dst[SZ_FNAME] + +begin + call clgstr ("source", df_src, SZ_FNAME) + call clgstr ("destination", df_dst, SZ_FNAME) + call fm_copy (df_src, df_dst) +end + + +# REBUILD -- Rebuild a datafile. + +procedure t_rebuild() + +char datafile[SZ_FNAME] + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + call fm_rebuild (datafile) +end + + +# Test the file cache package. +# ------------------------------- + +define GETFD 1 +define RETFD 2 +define LOCKOUT 3 +define UNLOCK 4 +define LOCKED 5 +define SYNC 6 +define DEBUG 7 +# +define FCDEBUG 9 +define PFILE 10 +define BYE 11 + +define KEYWORDS "|getfd|retfd|lockout|unlock|locked|sync|debug|\ + |fcdebug|pfile|bye|" + + +# FCACHE -- Test the file cache package. + +procedure t_fcache() + +pointer fm +int lfile, mode, type, fd +char datafile[SZ_FNAME], keyword[SZ_FNAME], junk[SZ_FNAME] +int strdic(), fscan(), fm_getfd() +bool fm_locked() +pointer fm_open() + +begin + call clgstr ("datafile", datafile, SZ_FNAME) + fm = fm_open (datafile, READ_WRITE) + + call printf ("* ") + call flush (STDOUT) + while (fscan (STDIN) != EOF) { + call gargwrd (keyword, SZ_FNAME) + if (IS_ALPHA(keyword[1])) + switch (strdic (keyword, junk, SZ_FNAME, KEYWORDS)) { + case GETFD: + call gargi (lfile) + call gargi (mode) + call gargi (type) + iferr (fd = fm_getfd (fm, lfile, mode, type)) + call erract (EA_WARN) + else { + call printf ("fd = %d\n") + call pargi (fd) + } + case RETFD: + call gargi (lfile) + call fm_retfd (fm, lfile) + + case LOCKOUT: + call gargi (lfile) + iferr (call fm_lockout (fm, lfile)) + call erract (EA_WARN) + case UNLOCK: + call gargi (lfile) + iferr (call fm_unlock (fm, lfile)) + call erract (EA_WARN) + case LOCKED: + call gargi (lfile) + call printf ("locked = %b\n") + call pargb (fm_locked (fm, lfile)) + + case SYNC: + call fm_fcsync (fm) + case DEBUG: + call fm_debug (fm, STDOUT, FMD_ALL) + case FCDEBUG: + call fm_fcdebug (fm, STDOUT, FCD_ALL) + case PFILE: + call gargi (lfile) + fd = fm_getfd (fm, lfile, READ_ONLY, TEXT_FILE) + iferr (call fcopyo (fd, STDOUT)) + call erract (EA_WARN) + call fm_retfd (fm, lfile) + case BYE: + break + default: + call eprintf ("commands: %s\n") + call pargstr (KEYWORDS) + } + + call printf ("* ") + call flush (STDOUT) + } + + call fm_close (fm) +end |