gf* check_shares_produced[self->mm - self->kk]; /* This is an upper bound -- we will actually use only num_check_shares_produced of these elements (see below). */
PyObject* pystrs_produced[self->mm - self->kk]; /* This is an upper bound -- we will actually use only num_check_shares_produced of these elements (see below). */
- unsigned num_check_shares_produced = 0; /* The first num_check_shares_produced elements of the check_shares_produced array and of the pystrs_produced array will be used. */
+ unsigned char num_check_shares_produced = 0; /* The first num_check_shares_produced elements of the check_shares_produced array and of the pystrs_produced array will be used. */
const gf* incshares[self->kk];
unsigned char num_desired_shares;
PyObject* fast_desired_shares_ids = NULL;
PyObject** fast_desired_shares_ids_items;
unsigned char c_desired_shares_ids[self->mm];
- unsigned i;
- unsigned prev_desired_id = 256; /* impossible value */
+ unsigned char c_desired_checkshares_ids[self->mm - self->kk];
+ unsigned char i;
if (desired_shares_ids) {
fast_desired_shares_ids = PySequence_Fast(desired_shares_ids, "Second argument (optional) was not a sequence.");
num_desired_shares = PySequence_Fast_GET_SIZE(fast_desired_shares_ids);
if (!PyInt_Check(fast_desired_shares_ids_items[i]))
goto err;
c_desired_shares_ids[i] = PyInt_AsLong(fast_desired_shares_ids_items[i]);
- if (prev_desired_id != 256 && prev_desired_id >= c_desired_shares_ids[i]) {
- py_raise_fec_error("Precondition violation: first argument is required to be in order -- each requested shareid in the sequence must be a higher number than the previous one. current requested shareid: %u, previous requested shareid: %u\n", c_desired_shares_ids[i], prev_desired_id);
- goto err;
- }
- prev_desired_id = c_desired_shares_ids[i];
if (c_desired_shares_ids[i] >= self->kk)
num_check_shares_produced++;
}
PyObject** fastinsharesitems = PySequence_Fast_ITEMS(fastinshares);
if (!fastinsharesitems)
goto err;
- int sz, oldsz = 0;
+ Py_ssize_t sz, oldsz = 0;
for (i=0; i<self->kk; i++) {
if (!PyObject_CheckReadBuffer(fastinsharesitems[i])) {
py_raise_fec_error("Precondition violation: %u'th item is required to offer the single-segment read character buffer protocol, but it does not.\n", i);
unsigned check_share_index = 0; /* index into the check_shares_produced and (parallel) pystrs_produced arrays */
for (i=0; i<num_desired_shares; i++) {
if (c_desired_shares_ids[i] >= self->kk) {
+ c_desired_checkshares_ids[check_share_index] = c_desired_shares_ids[i];
pystrs_produced[check_share_index] = PyString_FromStringAndSize(NULL, sz);
if (pystrs_produced[check_share_index] == NULL)
goto err;
assert (check_share_index == num_check_shares_produced);
/* Encode any check shares that are needed. */
- fec_encode(self->fec_matrix, incshares, check_shares_produced, c_desired_shares_ids, num_desired_shares, sz);
+ fec_encode(self->fec_matrix, incshares, check_shares_produced, c_desired_checkshares_ids, num_check_shares_produced, sz);
/* Wrap all requested shares up into a Python list of Python strings. */
result = PyList_New(num_desired_shares);
PyObject*restrict shareids;
PyObject* result = NULL;
- if (!PyArg_ParseTuple(args, "O!O!", &PyList_Type, &shares, &PyList_Type, &shareids))
+ if (!PyArg_ParseTuple(args, "OO", &shares, &shareids))
return NULL;
const gf*restrict cshares[self->kk];
unsigned char cshareids[self->kk];
gf*restrict recoveredcstrs[self->kk]; /* self->kk is actually an upper bound -- we probably won't need all of this space. */
PyObject*restrict recoveredpystrs[self->kk]; /* self->kk is actually an upper bound -- we probably won't need all of this space. */
- unsigned i;
+ unsigned char i;
for (i=0; i<self->kk; i++)
recoveredpystrs[i] = NULL;
PyObject*restrict fastshares = PySequence_Fast(shares, "First argument was not a sequence.");
}
/* Construct a C array of gf*'s of the data and another of C ints of the shareids. */
- unsigned needtorecover=0;
+ unsigned char needtorecover=0;
PyObject** fastshareidsitems = PySequence_Fast_ITEMS(fastshareids);
if (!fastshareidsitems)
goto err;
PyObject** fastsharesitems = PySequence_Fast_ITEMS(fastshares);
if (!fastsharesitems)
goto err;
- int sz, oldsz = 0;
+ Py_ssize_t sz, oldsz = 0;
for (i=0; i<self->kk; i++) {
if (!PyInt_Check(fastshareidsitems[i]))
goto err;
py_raise_fec_error("Precondition violation: %u'th item is required to offer the single-segment read character buffer protocol, but it does not.\n", i);
goto err;
}
- if (PyObject_AsReadBuffer(fastsharesitems[i], &(cshares[i]), &sz))
+ if (PyObject_AsReadBuffer(fastsharesitems[i], &(cshares[i]), &sz))
goto err;
if (oldsz != 0 && oldsz != sz) {
py_raise_fec_error("Precondition violation: Input shares are required to be all the same length. oldsz: %Zu, sz: %Zu\n", oldsz, sz);
fec_decode(self->fec_matrix, cshares, recoveredcstrs, cshareids, sz);
/* Wrap up both original primary shares and decoded shares into a Python list of Python strings. */
- unsigned nextrecoveredix=0;
+ unsigned char nextrecoveredix=0;
result = PyList_New(self->kk);
if (result == NULL)
goto err;
import fec
-import array
+import array, random
def encode_to_files(inf, prefix, k, m):
"""
Encode inf, writing the shares to named $prefix+$shareid.
"""
l = [ open(prefix+str(shareid), "wb") for shareid in range(m) ]
- def cb(shares, len):
+ def cb(shares, length):
assert len(shares) == len(l)
for i in range(len(shares)):
- l.write(share)
+ l[i].write(shares[i])
encode_file(inf, cb, k, m, chunksize=4096)
-def decode_from_files(outf, prefix, k, m):
+def decode_from_files(outf, filesize, prefix, k, m):
"""
Decode from the first k files in the current directory whose names begin
with prefix, writing the results to outf.
import os
infs = []
shareids = []
- for f in os.listdir("."):
+ listd = os.listdir(".")
+ random.shuffle(listd)
+ for f in listd:
if f.startswith(prefix):
infs.append(open(f, "rb"))
shareids.append(int(f[len(prefix):]))
x = [ inf.read(CHUNKSIZE) for inf in infs ]
decshares = dec.decode(x, shareids)
for decshare in decshares:
- outf.write(decshare)
- if len(x[-1]) != CHUNKSIZE:
- break
+ if filesize >= len(decshare):
+ outf.write(decshare)
+ filesize -= len(decshare)
+ else:
+ outf.write(decshare[:filesize])
+ return
def encode_file(inf, cb, k, m, chunksize=4096):
"""
# padding
a.fromstring("\x00" * (chunksize-len(a)))
while (i<len(l)):
+ a = l[i]
a[:] = ZEROES
i += 1
while indatasize == k*chunksize:
# This loop body executes once per segment.
i = 0
- l = []
+ l = []
ZEROES = '\x00'*chunksize
while i<k:
# This loop body executes once per chunk.
import fec
-import array
+import array, random
def bench_encode_to_files_shuffle_decode_from_files():
FILESIZE=1000000
print "Encoded %s byte file into %d share files in %0.2f seconds, or %0.2f million bytes per second" % (FILESIZE, M, so-st, FILESIZE/((so-st)*1000000),)
enctime = so-st
# Now delete m-k of the tempfiles at random.
- tempfs = [ f for f in os.listdir(".") if f.startwith(PREFIX) ]
- tempfs.shuffle()
+ tempfs = [ f for f in os.listdir(".") if f.startswith(PREFIX) ]
+ random.shuffle(tempfs)
for victimtempf in tempfs[:M-K]:
os.remove(victimtempf)
recoveredfile = open("tmpranddata-recovered", "wb")
st = time.time()
- fec.filefec.decode_from_files(recoveredfile, PREFIX, K, M)
+ fec.filefec.decode_from_files(recoveredfile, 1000000, PREFIX, K, M)
so = time.time()
- print "Encoded %s byte file from %d share files in %0.2f seconds, or %0.2f million bytes per second" % (FILESIZE, K, so-st, FILESIZE/((so-st)*1000000),)
+ print "Decoded %s byte file from %d share files in %0.2f seconds, or %0.2f million bytes per second" % (FILESIZE, K, so-st, FILESIZE/((so-st)*1000000),)
return enctime + (so-st)
finally:
# os.remove("tmpranddata")
projects / tahoe-lafs / zfec.git / commitdiff