checksum.c

Go to the documentation of this file.

/************************************************************************
 * Copyright (C) 2005-2009 Philipp Marek.
 *
 * This program is free software;  you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 3 as
 * published by the Free Software Foundation.
 ************************************************************************/

#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <apr_md5.h>
#include <sys/mman.h>

#include "checksum.h"
#include "helper.h"
#include "global.h"
#include "est_ops.h"
#include "waa.h"


#define MAPSIZE (256*1024*1024)



struct t_manber_parms
{
    AC_CV_C_UINT32_T values[256];
};

struct t_manber_data
{
    struct estat *sts;
    svn_stream_t *input;
    off_t last_fpos;
    off_t fpos;

    apr_md5_ctx_t full_md5_ctx;
    md5_digest_t full_md5;

    apr_md5_ctx_t block_md5_ctx;
    md5_digest_t block_md5;

    int manber_fd;


    AC_CV_C_UINT32_T state;
    AC_CV_C_UINT32_T last_state;
    int bktrk_bytes;
    int bktrk_last;
    unsigned char backtrack[CS__MANBER_BACKTRACK];
    int data_bits;
};

struct t_manber_parms manber_parms;


static struct t_manber_data cs___manber;


const char cs___mb_wr_format[]= "%s %08x %10llu %10llu\n";
const char cs___mb_rd_format[]= "%*s%n %x %llu %llu\n";

#define MANBER_LINELEN (APR_MD5_DIGESTSIZE*2+1 + 8+1 + 10+1 +10+1 + 1)


int cs___manber_data_init(struct t_manber_data *mbd, 
        struct estat *sts);
int cs___end_of_block(const unsigned char *data, int maxlen, 
        int *eob, 
        struct t_manber_data *mb_f);


inline static int cs__hex2val(char ch)
{
    /* I thought a bit whether I should store the values+1, ie. keep most of 
     * the array as 0 - but that doesn't save any memory, it only takes more 
     * time.
     * Sadly the various is*() functions (like isxdigit()) don't seem to 
     * include that information yet, and I couldn't find some table in the 
     * glibc sources.
     * (I couldn't find anything in that #define mess, TBH.) */
    static const signed char values[256]={
        /* 0x00 */
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        /* 0x20 = space ... */
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        /* 0x30 = "012345" */
        +0,  1,  2,  3,  4,  5,  6,  7,   8,  9, -1, -1, -1, -1, -1, -1,
        /* 0x40 = "@ABCD" ... */ 
        -1, 10, 11, 12, 13, 14, 15, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        /* 0x60 = "`abcd" ... */
        -1, 10, 11, 12, 13, 14, 15, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,

        /* 0x80 */
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,

        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
        -1, -1, -1, -1, -1, -1, -1, -1,  -1, -1, -1, -1, -1, -1, -1, -1,
    };

    /* To avoid having problems with "negative" characters */
    return values[ ch & 0xff ];
}


inline int cs__two_ch2bin(char *stg)
{
    return 
        (cs__hex2val(stg[0]) << 4) |
        (cs__hex2val(stg[1]) << 0);
}


int cs__char2md5(const char *input, char **eos, md5_digest_t md5)
{
    int i, status, x, y;


    status=0;
    for(i=0; i<APR_MD5_DIGESTSIZE; i++)
    {
        /* first check first character. If it's a \0, reading
         * the next character would possibly result in a SEGV. */
        x=cs__hex2val(input[0]);
        y=cs__hex2val(input[1]);

        /* I briefly thought about moving this test out of the loop, but that 
         * would possibly result in a SEGV if some (too short, but delimited) 
         * buffer gets parsed. */
        STOPIF_CODE_ERR( (y|x) == -1, EINVAL,
                "Illegal hex characters in %c%c", input[0], input[1]);

        input+=2;
        md5[i]=(x<<4) | y;
    }

    /* This should return a (const char*), to be compatible with the input.
     * But we cannot simply change all variables in ops__load_1entry() -
     * strtoul() has the same signature (const char*, char**). */
    if (eos) *eos=(char*)input;

ex:
    return status;
}


char* cs__md5tohex(const md5_digest_t md5, char *dest)
{
    /* According to callgrind is this one of the most cpu-intensive
     * places in *this* program (not counting libraries).
     * Of course, it vanishes against apr_file_read and others ...
     * But optimizing makes fun, and acta est fabula...
     *
     * If I read kcachegrind correctly, that shaved 1% runtime off on 
     * a simple "fsvs up".
     *
     * If there was only something like #while .... */
    sprintf(dest, 
            "%02x" "%02x" "%02x" "%02x" 
            "%02x" "%02x" "%02x" "%02x" 
            "%02x" "%02x" "%02x" "%02x" 
            "%02x" "%02x" "%02x" "%02x" 
            ,
            md5[4*0+0], md5[4*0+1], md5[4*0+2], md5[4*0+3], 
            md5[4*1+0], md5[4*1+1], md5[4*1+2], md5[4*1+3], 
            md5[4*2+0], md5[4*2+1], md5[4*2+2], md5[4*2+3], 
            md5[4*3+0], md5[4*3+1], md5[4*3+2], md5[4*3+3]
            );
#if 0
    int i;

    // Simple, slow variant
    for (i=0; i<APR_MD5_DIGESTSIZE; i++)
    {
        sprintf(cur+i*2, "%02x", md5[i]);
    }
#endif

    return dest;
}


char *cs__md5tohex_buffered(const md5_digest_t md5)
{
    static int last=0;
    static char stg[4][APR_MD5_DIGESTSIZE*2+1] = { { 0 } };
    char *cur;

    last++;
    if (last >= sizeof(stg)/sizeof(stg[0])) last=0;
    cur=stg[last];

    return cs__md5tohex(md5, cur);
}


int cs___finish_manber(struct t_manber_data *mb_f)
{
    int status;

    status=0;
    STOPIF( apr_md5_final(mb_f->full_md5, & mb_f->full_md5_ctx), 
            "apr_md5_final failed");
    if (mb_f->sts)
        memcpy(mb_f->sts->md5, mb_f->full_md5, sizeof(mb_f->sts->md5));

    mb_f->sts=NULL;

ex:
    return status;
}


int cs__compare_file(struct estat *sts, char *fullpath, int *result)
{
    int i, status, fh;
    unsigned length_mapped, map_pos, hash_pos;
    off_t current_pos;
    struct cs__manber_hashes mbh_data;
    unsigned char *filedata;
    int do_manber;
    char *cp;
    struct sstat_t actual;
    md5_digest_t old_md5 = { 0 };
    static struct t_manber_data mb_dat;


    /* Default is "don't know". */
    if (result) *result = -1;
    /* It doesn't matter whether we test this or old_rev_mode_packed - if 
     * they're different, this entry was replaced, and we never get here.  */
    if (S_ISDIR(sts->st.mode)) return 0;

    fh=-1;
    /* hash already done? */
    if (sts->change_flag != CF_UNKNOWN)
    {
        DEBUGP("change flag for %s: %d", fullpath, sts->change_flag);
        goto ret_result;
    }

    status=0;

    if (!fullpath)
        STOPIF( ops__build_path(&fullpath, sts), NULL);

    DEBUGP("checking for modification on %s", fullpath);

    DEBUGP("hashing %s",fullpath);
    memcpy(old_md5, sts->md5, sizeof(old_md5));

    /* We'll open and read the file now, so the additional lstat() doesn't 
     * really hurt - and it makes sure that we see the current values (or at 
     * least the _current_ ones :-). */
    STOPIF( hlp__lstat(fullpath, &actual), NULL);

    if (S_ISREG(actual.mode))
    {
        do_manber=1;
        /* Open the file and read the stream from there, comparing the blocks
         * as necessary.
         * If a difference is found, stop, and mark file as different. */
        /* If this call returns ENOENT, this entry simply has no md5s-file.
         * We'll have to MD5 it completely. */
        if (actual.size < CS__MIN_FILE_SIZE)
            do_manber=0;
        else
        {
            status=cs__read_manber_hashes(sts, &mbh_data);
            if (status == ENOENT)
                do_manber=0;
            else 
                STOPIF(status, "reading manber-hash data for %s", fullpath);
        }

        hash_pos=0;
        STOPIF( cs___manber_data_init(&mb_dat, sts), NULL );

        /* We map windows of the file into main memory. Never more than 256MB. */
        current_pos=0;

        fh=open(fullpath, O_RDONLY);
        /* We allow a single special case on error handling: EACCES, which 
         * could simply mean that the file has mode 000. */
        if (fh<0)
        {
            /* The debug statement might change errno, so we have to save the 
             * value.  */
            status=errno;
            DEBUGP("File %s is unreadable: %d", fullpath, status);
            if (status == EACCES) 
            {
                status=0;
                goto ex;
            }

            /* Can that happen? */
            if (!status) status=EBUSY;
            STOPIF(status, "open(\"%s\", O_RDONLY) failed", fullpath);
        }

        status=0;
        while (current_pos < actual.size)
        {
            if (actual.size-current_pos < MAPSIZE)
                length_mapped=actual.size-current_pos;
            else
                length_mapped=MAPSIZE;
            DEBUGP("mapping %u bytes from %llu", 
                    length_mapped, (t_ull)current_pos); 

            filedata=mmap(NULL, length_mapped, 
                    PROT_READ, MAP_SHARED, 
                    fh, current_pos);
            if (filedata == MAP_FAILED) break;

            map_pos=0;
            while (map_pos<length_mapped)
            {
                STOPIF( cs___end_of_block(filedata+map_pos,
                            length_mapped-map_pos, 
                            &i, &mb_dat ),
                        NULL);

                if (i==-1) break;

                if (do_manber)
                {
                    DEBUGP("  old hash=%08X  current hash=%08X", 
                            mbh_data.hash[hash_pos], mb_dat.last_state);
                    DEBUGP("  old end=%llu  current end=%llu", 
                            (t_ull)mbh_data.end[hash_pos], 
                            (t_ull)mb_dat.fpos);
                    DEBUGP("  old md5=%s  current md5=%s", 
                            cs__md5tohex_buffered(mbh_data.md5[hash_pos]),
                            cs__md5tohex_buffered(mb_dat.block_md5));

                    if (mb_dat.last_state != mbh_data.hash[hash_pos] ||
                            mb_dat.fpos != mbh_data.end[hash_pos] ||
                            memcmp(mb_dat.block_md5, 
                                mbh_data.md5[hash_pos], 
                                APR_MD5_DIGESTSIZE) != 0)
                    {
                        DEBUGP("found a different block before %llu:", 
                                (t_ull)(current_pos+i));
changed:
                        sts->md5[0] ^= 0x1;
                        i=-2;
                        break;
                    }


                    DEBUGP("block #%u ok...", hash_pos);
                    hash_pos++;
                    /* If this gets true (which it should never), we must not
                     * print the hash values etc. ... The index [hash_pos] is outside
                     * the array boundaries. */
                    if (hash_pos > mbh_data.count)
                        goto changed;
                }

                /* We have to reset the blocks even if we have no manber hashes ...  
                 * so the eg. data_bits value gets reset. */
                STOPIF( cs___end_of_block(NULL, 0, NULL, &mb_dat), NULL );

                map_pos+=i;
            }
            if (i==-2) break;

            STOPIF_CODE_ERR( munmap((void*)filedata, length_mapped) == -1,
                    errno, "unmapping of file failed");
            current_pos+=length_mapped;
        }

        STOPIF( cs___finish_manber( &mb_dat), NULL);
    }
    else if (S_ISLNK(sts->st.mode))
    {
        STOPIF( ops__link_to_string(sts, fullpath, &cp), NULL);

        apr_md5(sts->md5, cp, strlen(cp));
    }
    else
    {
        DEBUGP("nothing to hash for %s", fullpath);
    }

    sts->change_flag = memcmp(old_md5, sts->md5, sizeof(sts->md5)) == 0 ?
        CF_NOTCHANGED : CF_CHANGED;
    DEBUGP("change flag for %s set to %d", fullpath, sts->change_flag);


ret_result:
    if (result)
        *result = sts->change_flag == CF_CHANGED;
    DEBUGP("comparing %s=%d: md5 %s", 
            fullpath, sts->change_flag == CF_CHANGED,
            cs__md5tohex_buffered(sts->md5));
    status=0;

ex:
    if (fh>=0) close(fh);

    return status;
}


int cs__set_file_committed(struct estat *sts) 
{
    int status;

    status=0;
    if (S_ISDIR(sts->st.mode)) goto ex;

    /* Now we can drop the check flag. */
    sts->flags &= ~(RF_CHECK | RF_PUSHPROPS);

    sts->repos_rev=SET_REVNUM;

ex:
    return status;
}


/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
 * Stream functions and callbacks
 * for manber-filtering
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

int cs___manber_data_init(struct t_manber_data *mbd, 
        struct estat *sts)
{
    int status;

    memset(mbd, 0, sizeof(*mbd));
    mbd->manber_fd=-1;

    BUG_ON(mbd->sts, "manber structure already in use!");
    mbd->sts=sts;
    mbd->fpos= mbd->last_fpos= 0;
    apr_md5_init(& mbd->full_md5_ctx);
    STOPIF( cs___end_of_block(NULL, 0, NULL, mbd), NULL );

ex:
    return status;
}


void cs___manber_init(struct t_manber_parms *mb_d)
{
    int i;
    AC_CV_C_UINT32_T p;

    if (mb_d->values[0]) return;

    /* Calculate the CS__MANBER_BACKTRACK power of the prime */
    /* TODO: speedup like done in RSA - log2(power) */
    for(p=1,i=0; i<CS__MANBER_BACKTRACK; i++)
        p=(p * CS__MANBER_PRIME) & CS__MANBER_MODULUS;

    /* Precalculate for all 8bit values */
    for(i=0x00; i<0xff; i++)
        mb_d->values[i]=(i*p) & CS__MANBER_MODULUS;
}


/* This function finds the position which 
 *
 *   a b c d e f g h i j k l m n
 *   |..Block.1..| |..Block.2...
 * Here it would return h; ie. the number of characters
 * found in this data block belonging to the current block.
 *
 * If the whole data buffer belongs to the current block -1 is returned
 * in *eob.
 * */
int cs___end_of_block(const unsigned char *data, int maxlen, 
        int *eob, 
        struct t_manber_data *mb_f)
{
    int status;
    int i;


    status=0;
    if (!data)
    {
        DEBUGP("manber reinit");
        mb_f->state=0;
        mb_f->last_state=0;
        mb_f->bktrk_bytes=0;
        mb_f->bktrk_last=0;
        mb_f->data_bits=0;
        apr_md5_init(& mb_f->block_md5_ctx);
        memset(mb_f->block_md5, 0, sizeof(mb_f->block_md5));
        cs___manber_init(&manber_parms);
        goto ex;
    }


    *eob = -1;
    i=0;
    /* If we haven't had at least this many bytes in the current block,
     * read up to this amount. */
    while (i<maxlen &&
            mb_f->bktrk_bytes < CS__MANBER_BACKTRACK)
    {
        /* In this initialization, we simply \c OR the bytes together.
         * On block end detection we see if this is at least a
         * \c CS__MANBER_BACKTRACK bytes long zero-byte block. */
        mb_f->data_bits |= data[i];

        mb_f->state = (mb_f->state * CS__MANBER_PRIME +
                data[i] ) % CS__MANBER_MODULUS;
        mb_f->backtrack[ mb_f->bktrk_last ] = data[i];
        /* The reason why CS__MANBER_BACKTRACK must be a power of 2:
         * bitwise-AND is much faster than a modulo.
         * In this loop the & is redundant - in a new block we should
         * start from bktrk_last==0, but the AND is only 1 or 2 cycles,
         * and we hope that gcc optimizes that. */
        mb_f->bktrk_last = ( mb_f->bktrk_last + 1 ) & 
            (CS__MANBER_BACKTRACK - 1);
        mb_f->bktrk_bytes++;
        i++;
    }

    if (!mb_f->data_bits)
    {
        /* No bits in the data set - only zeroes so far.
         * Look for the next non-zero byte; there's a block border. */
        /* memchr is the exact opposite of what we need. */
        while (i<maxlen && !data[i]) i++;

        if (i < maxlen)
        {
            *eob=i;
            DEBUGP("zero block border at %d", i);
        }
    }
    else
    {
        while(i<maxlen)
        {
            /* ->last_state gets the previous CRC, and this gets stored.
             * This is because the ->state has, on a block border, a lot of
             * zeroes (per definition); so we store the previous value, which
             * may be better suited for comparison. If the blocks are equal 
             * up to byte N, they're equal up to N-1, too. */
            /* This need not be calculated in the previous loop, as we do no
             * border-checking there. Only here, in this loop, 
             * is the value needed. */
            mb_f->last_state=mb_f->state;
            mb_f->state = (mb_f->state*CS__MANBER_PRIME + data[i] -
                    manber_parms.values[ mb_f->backtrack[ mb_f->bktrk_last ] ] ) 
                % CS__MANBER_MODULUS;
            mb_f->backtrack[ mb_f->bktrk_last ] = data[i];
            mb_f->bktrk_last = ( mb_f->bktrk_last + 1 ) & 
                (CS__MANBER_BACKTRACK - 1);
            /* This value has already been used. */
            i++;

            /* special value ? */
            if ( !(mb_f->state & CS__MANBER_BITMASK) )
            {
                *eob=i;
                apr_md5_update(& mb_f->block_md5_ctx, data, i);
                apr_md5_final( mb_f->block_md5, & mb_f->block_md5_ctx);
                DEBUGP("manber found a border: %u %08X %08X %s", 
                        i, mb_f->last_state, mb_f->state, cs__md5tohex_buffered(mb_f->block_md5));
                break;
            }
        }

        /* Update md5 up to current byte. */
        if (*eob == -1)
            apr_md5_update(& mb_f->block_md5_ctx, data, i);
    }

    /* Update file global information */
    apr_md5_update(& mb_f->full_md5_ctx, data, i);
    mb_f->fpos += (*eob == -1) ? maxlen : *eob;

ex:
    DEBUGP("on return at fpos=%llu: %08X (databits=%2x)", 
            (t_ull)mb_f->fpos, mb_f->state, mb_f->data_bits);
    return status;
}


int cs___update_manber(struct t_manber_data *mb_f,
        const unsigned char *data, apr_size_t len)
{
    int status;
    int eob, i;
    /* MD5 as hex (constant-length), 
     * state as hex (constant-length),
     * offset of block, length of block, 
     * \n, \0, reserve */
    char buffer[MANBER_LINELEN+10];
    char *filename;

    status=0;
    /* We tried to avoid doing this calculation for small files.
     *
     * But: this does not work.
     * As *on update* we don't know how many bytes we'll
     * have to process, and the buffer size is not specified,
     * we might be legitimately called with single-byte values.
     *
     * On my machine I get for commit/update requests of 100kB 
     * (yes, 102400 bytes), so I thought to have at least some chance.
     * But: svn_ra_get_file uses 16k blocks ... 
     *
     * A full solution would either have to
     *   - know how many bytes we get (update; on commit we know), or to
     *   - buffer all to-be-written-data unless we have more bytes
     *     than limited.
     * The second variant might be easier.
     *
     * As we now check on end-of-stream for the size and remove the file if
     * necessary, this is currently deactivated. */
    DEBUGP("got a block with %llu bytes", (t_ull)len);
    while (1)
    {
#if 0
        /* If first call, and buffer smaller than 32kB, avoid this calling ... */
        if (mb_f->fpos == 0 && len<32*1024)
            eob=-1;
        else
#endif
            STOPIF( cs___end_of_block(data, len, &eob, mb_f), NULL );

        if (eob == -1) 
        {
            DEBUGP("block continues after %lu.", (unsigned long)mb_f->fpos);
            break;
        }

        data += eob;
        len -= eob;
        DEBUGP("block ends after %lu; size %lu bytes (border=%u).", 
                (unsigned long)mb_f->fpos,
                (unsigned long)(mb_f->fpos - mb_f->last_fpos),
                eob);

        /* write new line to data file */
        i=sprintf(buffer, cs___mb_wr_format, 
                cs__md5tohex_buffered(mb_f->block_md5),
                mb_f->last_state,
                (t_ull)mb_f->last_fpos, 
                (t_ull)(mb_f->fpos - mb_f->last_fpos));
        BUG_ON(i > sizeof(buffer)-3, "Buffer too small - stack overrun");

        if (mb_f->manber_fd == -1)
        {
            /* The file has not been opened yet.
             * Do it now.
             * */
            STOPIF( ops__build_path(&filename, mb_f->sts), NULL);
            STOPIF( waa__open_byext(filename, WAA__FILE_MD5s_EXT, WAA__WRITE,
                        &   cs___manber.manber_fd), NULL );
            DEBUGP("now doing manber-hashing for %s...", filename);
        }

        STOPIF_CODE_ERR( write( mb_f->manber_fd, buffer, i) != i,
                errno, "writing to manber hash file");

        /* re-init manber state */
        STOPIF( cs___end_of_block(NULL, 0, NULL, mb_f), NULL );
        mb_f->last_fpos = mb_f->fpos;
    }

ex:
    return status;
}


svn_error_t *cs___mnbs_close(void *baton);


svn_error_t *cs___mnbs_read(void *baton, 
        char *data, apr_size_t *len)
{
    int status;
    svn_error_t *status_svn;
    struct t_manber_data *mb_f=baton;

    status=0;
    /* Get the bytes, then process them. */
    STOPIF_SVNERR( svn_stream_read,
            (mb_f->input, data, len) );
    if (*len && data)
        STOPIF( cs___update_manber(mb_f, (unsigned char*)data, *len), NULL);
    else
        STOPIF_SVNERR( cs___mnbs_close, (baton));
ex:
    RETURN_SVNERR(status);
}


svn_error_t *cs___mnbs_write(void *baton, 
        const char *data, apr_size_t *len)
{
    int status;
    svn_error_t *status_svn;
    struct t_manber_data *mb_f=baton;

    status=0;
    /* We first write to the output stream, to know how many bytes could
     * be processed. Then we use that bytes. 
     * If we just processed the incoming bytes, then fewer would get written,
     * and the remaining would be re-done we'd hash them twice. */
    STOPIF_SVNERR( svn_stream_write,
            (mb_f->input, data, len) );
    if (*len && data)
        STOPIF( cs___update_manber(mb_f, (unsigned char*)data, *len), NULL);
    else
        STOPIF_SVNERR( cs___mnbs_close, (baton));
ex:
    RETURN_SVNERR(status);
}


svn_error_t *cs___mnbs_close(void *baton)
{
    int status;
    svn_error_t *status_svn;
    struct t_manber_data *mb_f=baton;

    status=0;

    /* If there have been less than CS__MIN_FILE_SIZE bytes, we
     * don't keep that file. */
    if (mb_f->manber_fd != -1)
    {
        STOPIF( waa__close(mb_f->manber_fd, 
                    mb_f->fpos < CS__MIN_FILE_SIZE ? ECANCELED : 
                    status != 0), NULL );
        mb_f->manber_fd=-1;
    }

    if (mb_f->input)
    {
        STOPIF_SVNERR( svn_stream_close,
                (mb_f->input) );
        mb_f->input=NULL;
    }

    STOPIF( cs___finish_manber(mb_f), NULL);

ex:
    RETURN_SVNERR(status);
}


int cs__new_manber_filter(struct estat *sts,
        svn_stream_t *stream_input, 
        svn_stream_t **filter_stream,
        apr_pool_t *pool)
{
    int status;
    svn_stream_t *new_str;
    char *filename;


    status=0;
    STOPIF( cs___manber_data_init(&cs___manber, sts),
            "manber-data-init failed");

    cs___manber.input=stream_input;

    new_str=svn_stream_create(&cs___manber, pool);
    STOPIF_ENOMEM( !new_str );

    svn_stream_set_read(new_str, cs___mnbs_read);
    svn_stream_set_write(new_str, cs___mnbs_write);
    svn_stream_set_close(new_str, cs___mnbs_close);

    STOPIF( ops__build_path(&filename, sts), NULL);
    DEBUGP("initiating MD5 streaming for %s", filename);

    *filter_stream=new_str;

    /* The file with the hashes for the blocks is not immediately opened.
     * only when we detect that we have at least a minimum file size
     * we do the whole calculation.*/

ex:
    return status;
}


int cs__read_manber_hashes(struct estat *sts, struct cs__manber_hashes *data)
{
    int status;
    char *filename;
    int fh, i, spp;
    unsigned estimated, count;
    t_ull length, start;
    char buffer[MANBER_LINELEN+10], *cp;
    AC_CV_C_UINT32_T value;


    status=0;
    memset(data, 0, sizeof(*data));
    fh=-1;

    STOPIF( ops__build_path(&filename, sts), NULL);
    /* It's ok if there's no md5s file. simply return ENOENT. */
    status=waa__open_byext(filename, WAA__FILE_MD5s_EXT, WAA__READ, &fh);
    if (status == ENOENT) goto ex;
    STOPIF( status, "reading md5s-file for %s", filename);

    DEBUGP("reading manber-hashes for %s", filename);

    /* We don't know in advance how many lines (i.e. manber-hashes)
     * there will be.
     * So we just interpolate from the file size and the (near-constant)
     * line-length and add a bit for good measure.
     * The rest is freed as soon as we've got all entries. */
    length=lseek(fh, 0, SEEK_END);
    STOPIF_CODE_ERR( length==-1, errno, 
            "Cannot get length of file %s", filename);
    STOPIF_CODE_ERR( lseek(fh, 0, SEEK_SET), errno, 
            "Cannot seek in file %s", filename);
    /* We add 5%; due to integer arithmetic the factors have to be separated */
    estimated = length*21/(MANBER_LINELEN*20)+4;
    DEBUGP("estimated %u manber-hashes from filelen %llu", 
            estimated, length);

    /* Allocate memory ... */
    STOPIF( hlp__calloc( &data->hash, estimated, sizeof(*data->hash)), NULL);
    STOPIF( hlp__calloc( & data->md5, estimated, sizeof( *data->md5)), NULL);
    STOPIF( hlp__calloc( & data->end, estimated, sizeof( *data->end)), NULL);


    count=0;
    while (1)
    {
        i=read(fh, buffer, sizeof(buffer)-1);
        STOPIF_CODE_ERR( i==-1, errno,
                "reading manber-hash data");
        if (i==0) break;

        /* ensure strchr() stops */
        buffer[i]=0;
        cp=strchr(buffer, '\n');
        STOPIF_CODE_ERR(!cp, EINVAL, 
                "line %u is invalid", count+1 );

        /* reposition to start of next line */
        STOPIF_CODE_ERR( lseek(fh, 
                    -i                              // start of this line
                    + (cp-buffer)           // end of this line
                    + 1,                            // over \n
                    SEEK_CUR) == -1, errno, 
                "lseek back failed");
        *cp=0;

        i=sscanf(buffer, cs___mb_rd_format,
                &spp, &value, &start, &length);
        STOPIF_CODE_ERR( i != 3, EINVAL,
                "cannot parse line %u for %s", count+1, filename);

        data->hash[count]=value;
        data->end[count]=start+length;
        buffer[spp]=0;
        STOPIF( cs__char2md5(buffer, NULL, data->md5[count]), NULL);
        count++;
        BUG_ON(count > estimated, "lines should have syntax errors - bug in estimation.");
    }

    data->count=count;
    DEBUGP("read %u entry tuples.", count);

    if (estimated-count > 3)
    {
        DEBUGP("reallocating...");
        /* reallocate memory = free */
        STOPIF( hlp__realloc( &data->hash, count*sizeof(*data->hash)), NULL);
        STOPIF( hlp__realloc( & data->md5, count*sizeof(* data->md5)), NULL);
        STOPIF( hlp__realloc( & data->end, count*sizeof(* data->end)), NULL);
    }

    /* The index is not always needed. Don't generate now. */

ex:
    if (status)
    {
        IF_FREE(data->hash);
        IF_FREE(data->md5);
        IF_FREE(data->end);
    }

    if (fh != -1)
        STOPIF_CODE_ERR( close(fh) == -1, errno, 
                "Cannot close manber hash file (fd=%d)", fh);
    return status;
}

---