2 * zfec -- fast forward error correction library with Python interface
6 #include <structmember.h>
8 #if (PY_VERSION_HEX < 0x02050000)
9 typedef int Py_ssize_t;
17 // actually, some of the flavors (i.e. Enterprise) do support restrict
18 //#define restrict __restrict
20 #define inline __inline
21 #define alloca _alloca
24 #define alloca(x) __builtin_alloca(x)
32 static PyObject *py_fec_error;
34 static char fec__doc__[] = "\
35 FEC - Forward Error Correction \n\
38 static char Encoder__doc__[] = "\
39 Hold static encoder state (an in-memory table for matrix multiplication), and k and m parameters, and provide {encode()} method.\n\n\
40 @param k: the number of packets required for reconstruction \n\
41 @param m: the number of packets generated \n\
56 Encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwdict) {
59 self = (Encoder*)type->tp_alloc(type, 0);
63 self->fec_matrix = NULL;
66 return (PyObject *)self;
70 Encoder_init(Encoder *self, PyObject *args, PyObject *kwdict) {
71 static char *kwlist[] = {
77 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "ii", kwlist, &ink, &inm))
81 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be greater than or equal to 1, but it was %d", self->kk);
85 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be greater than or equal to 1, but it was %d", self->mm);
89 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be less than or equal to 256, but it was %d", self->mm);
93 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be less than or equal to the second argument, but they were %d and %d respectively", ink, inm);
96 self->kk = (short)ink;
97 self->mm = (short)inm;
98 self->fec_matrix = fec_new(self->kk, self->mm);
103 static char Encoder_encode__doc__[] = "\
104 Encode data into m packets.\n\
106 @param inblocks: a sequence of k buffers of data to encode -- these are the k primary blocks, i.e. the input data split into k pieces (for best performance, make it a tuple instead of a list); All blocks are required to be the same length.\n\
107 @param desired_blocks_nums optional sequence of blocknums indicating which blocks to produce and return; If None, all m blocks will be returned (in order). (For best performance, make it a tuple instead of a list.)\n\
108 @returns: a list of buffers containing the requested blocks; Note that if any of the input blocks were 'primary blocks', i.e. their blocknum was < k, then the result sequence will contain a Python reference to the same Python object as was passed in. As long as the Python object in question is immutable (i.e. a string) then you don't have to think about this detail, but if it is mutable (i.e. an array), then you have to be aware that if you subsequently mutate the contents of that object then that will also change the contents of the sequence that was returned from this call to encode().\n\
112 Encoder_encode(Encoder *self, PyObject *args) {
114 PyObject* desired_blocks_nums = NULL; /* The blocknums of the blocks that should be returned. */
115 PyObject* result = NULL;
117 gf** check_blocks_produced = (gf**)alloca((self->mm - self->kk) * sizeof(PyObject*)); /* This is an upper bound -- we will actually use only num_check_blocks_produced of these elements (see below). */
118 PyObject** pystrs_produced = (PyObject**)alloca((self->mm - self->kk) * sizeof(PyObject*)); /* This is an upper bound -- we will actually use only num_check_blocks_produced of these elements (see below). */
119 unsigned num_check_blocks_produced = 0; /* The first num_check_blocks_produced elements of the check_blocks_produced array and of the pystrs_produced array will be used. */
120 const gf** incblocks = (const gf**)alloca(self->kk * sizeof(const gf*));
121 unsigned num_desired_blocks;
122 PyObject* fast_desired_blocks_nums = NULL;
123 PyObject** fast_desired_blocks_nums_items;
124 unsigned* c_desired_blocks_nums = (unsigned*)alloca(self->mm * sizeof(unsigned));
125 unsigned* c_desired_checkblocks_ids = (unsigned*)alloca((self->mm - self->kk) * sizeof(unsigned));
127 PyObject* fastinblocks = NULL;
128 PyObject** fastinblocksitems;
129 Py_ssize_t sz, oldsz = 0;
130 unsigned char check_block_index = 0; /* index into the check_blocks_produced and (parallel) pystrs_produced arrays */
132 if (!PyArg_ParseTuple(args, "O|O", &inblocks, &desired_blocks_nums))
135 for (i=0; i<self->mm - self->kk; i++)
136 pystrs_produced[i] = NULL;
137 if (desired_blocks_nums) {
138 fast_desired_blocks_nums = PySequence_Fast(desired_blocks_nums, "Second argument (optional) was not a sequence.");
139 if (!fast_desired_blocks_nums)
141 num_desired_blocks = PySequence_Fast_GET_SIZE(fast_desired_blocks_nums);
142 fast_desired_blocks_nums_items = PySequence_Fast_ITEMS(fast_desired_blocks_nums);
143 for (i=0; i<num_desired_blocks; i++) {
144 if (!PyInt_Check(fast_desired_blocks_nums_items[i])) {
145 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to contain int.");
148 c_desired_blocks_nums[i] = PyInt_AsLong(fast_desired_blocks_nums_items[i]);
149 if (c_desired_blocks_nums[i] >= self->kk)
150 num_check_blocks_produced++;
153 num_desired_blocks = self->mm;
154 for (i=0; i<num_desired_blocks; i++)
155 c_desired_blocks_nums[i] = i;
156 num_check_blocks_produced = self->mm - self->kk;
159 fastinblocks = PySequence_Fast(inblocks, "First argument was not a sequence.");
163 if (PySequence_Fast_GET_SIZE(fastinblocks) != self->kk) {
164 PyErr_Format(py_fec_error, "Precondition violation: Wrong length -- first argument (the sequence of input blocks) is required to contain exactly k blocks. len(first): %d, k: %d", PySequence_Fast_GET_SIZE(fastinblocks), self->kk);
168 /* Construct a C array of gf*'s of the input data. */
169 fastinblocksitems = PySequence_Fast_ITEMS(fastinblocks);
170 if (!fastinblocksitems)
173 for (i=0; i<self->kk; i++) {
174 if (!PyObject_CheckReadBuffer(fastinblocksitems[i])) {
175 PyErr_Format(py_fec_error, "Precondition violation: %u'th item is required to offer the single-segment read character buffer protocol, but it does not.", i);
178 if (PyObject_AsReadBuffer(fastinblocksitems[i], (const void**)&(incblocks[i]), &sz))
180 if (oldsz != 0 && oldsz != sz) {
181 PyErr_Format(py_fec_error, "Precondition violation: Input blocks are required to be all the same length. length of one block was: %Zu, length of another block was: %Zu", oldsz, sz);
187 /* Allocate space for all of the check blocks. */
189 for (i=0; i<num_desired_blocks; i++) {
190 if (c_desired_blocks_nums[i] >= self->kk) {
191 c_desired_checkblocks_ids[check_block_index] = c_desired_blocks_nums[i];
192 pystrs_produced[check_block_index] = PyString_FromStringAndSize(NULL, sz);
193 if (pystrs_produced[check_block_index] == NULL)
195 check_blocks_produced[check_block_index] = (gf*)PyString_AsString(pystrs_produced[check_block_index]);
196 if (check_blocks_produced[check_block_index] == NULL)
201 assert (check_block_index == num_check_blocks_produced);
203 /* Encode any check blocks that are needed. */
204 fec_encode(self->fec_matrix, incblocks, check_blocks_produced, c_desired_checkblocks_ids, num_check_blocks_produced, sz);
206 /* Wrap all requested blocks up into a Python list of Python strings. */
207 result = PyList_New(num_desired_blocks);
210 check_block_index = 0;
211 for (i=0; i<num_desired_blocks; i++) {
212 if (c_desired_blocks_nums[i] < self->kk) {
213 Py_INCREF(fastinblocksitems[c_desired_blocks_nums[i]]);
214 if (PyList_SetItem(result, i, fastinblocksitems[c_desired_blocks_nums[i]]) == -1) {
215 Py_DECREF(fastinblocksitems[c_desired_blocks_nums[i]]);
219 if (PyList_SetItem(result, i, pystrs_produced[check_block_index]) == -1)
221 pystrs_produced[check_block_index] = NULL;
228 for (i=0; i<num_check_blocks_produced; i++)
229 Py_XDECREF(pystrs_produced[i]);
230 Py_XDECREF(result); result = NULL;
232 Py_XDECREF(fastinblocks); fastinblocks=NULL;
233 Py_XDECREF(fast_desired_blocks_nums); fast_desired_blocks_nums=NULL;
238 Encoder_dealloc(Encoder * self) {
239 fec_free(self->fec_matrix);
240 self->ob_type->tp_free((PyObject*)self);
243 static PyMethodDef Encoder_methods[] = {
244 {"encode", (PyCFunction)Encoder_encode, METH_VARARGS, Encoder_encode__doc__},
248 static PyMemberDef Encoder_members[] = {
249 {"k", T_SHORT, offsetof(Encoder, kk), READONLY, "k"},
250 {"m", T_SHORT, offsetof(Encoder, mm), READONLY, "m"},
251 {NULL} /* Sentinel */
254 static PyTypeObject Encoder_type = {
255 PyObject_HEAD_INIT(NULL)
257 "_fec.Encoder", /*tp_name*/
258 sizeof(Encoder), /*tp_basicsize*/
260 (destructor)Encoder_dealloc, /*tp_dealloc*/
267 0, /*tp_as_sequence*/
275 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
276 Encoder__doc__, /* tp_doc */
279 0, /* tp_richcompare */
280 0, /* tp_weaklistoffset */
283 Encoder_methods, /* tp_methods */
284 Encoder_members, /* tp_members */
288 0, /* tp_descr_get */
289 0, /* tp_descr_set */
290 0, /* tp_dictoffset */
291 (initproc)Encoder_init, /* tp_init */
293 Encoder_new, /* tp_new */
296 static char Decoder__doc__[] = "\
297 Hold static decoder state (an in-memory table for matrix multiplication), and k and m parameters, and provide {decode()} method.\n\n\
298 @param k: the number of packets required for reconstruction \n\
299 @param m: the number of packets generated \n\
314 Decoder_new(PyTypeObject *type, PyObject *args, PyObject *kwdict) {
317 self = (Decoder*)type->tp_alloc(type, 0);
321 self->fec_matrix = NULL;
324 return (PyObject *)self;
328 Decoder_init(Encoder *self, PyObject *args, PyObject *kwdict) {
329 static char *kwlist[] = {
336 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "ii", kwlist, &ink, &inm))
340 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be greater than or equal to 1, but it was %d", self->kk);
344 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be greater than or equal to 1, but it was %d", self->mm);
348 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be less than or equal to 256, but it was %d", self->mm);
352 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be less than or equal to the second argument, but they were %d and %d respectively", ink, inm);
355 self->kk = (short)ink;
356 self->mm = (short)inm;
357 self->fec_matrix = fec_new(self->kk, self->mm);
362 #define SWAP(a,b,t) {t tmp; tmp=a; a=b; b=tmp;}
364 static char Decoder_decode__doc__[] = "\
365 Decode a list blocks into a list of segments.\n\
366 @param blocks a sequence of buffers containing block data (for best performance, make it a tuple instead of a list)\n\
367 @param blocknums a sequence of integers of the blocknum for each block in blocks (for best performance, make it a tuple instead of a list)\n\
369 @return a list of strings containing the segment data (i.e. ''.join(retval) yields a string containing the decoded data)\n\
373 Decoder_decode(Decoder *self, PyObject *args) {
374 PyObject*restrict blocks;
375 PyObject*restrict blocknums;
376 PyObject* result = NULL;
378 const gf**restrict cblocks = (const gf**restrict)alloca(self->kk * sizeof(const gf*));
379 unsigned* cblocknums = (unsigned*)alloca(self->kk * sizeof(unsigned));
380 gf**restrict recoveredcstrs = (gf**)alloca(self->kk * sizeof(gf*)); /* self->kk is actually an upper bound -- we probably won't need all of this space. */
381 PyObject**restrict recoveredpystrs = (PyObject**restrict)alloca(self->kk * sizeof(PyObject*)); /* self->kk is actually an upper bound -- we probably won't need all of this space. */
383 PyObject*restrict fastblocknums = NULL;
384 PyObject*restrict fastblocks;
385 unsigned needtorecover=0;
386 PyObject** fastblocksitems;
387 PyObject** fastblocknumsitems;
388 Py_ssize_t sz, oldsz = 0;
390 unsigned nextrecoveredix=0;
392 if (!PyArg_ParseTuple(args, "OO", &blocks, &blocknums))
395 for (i=0; i<self->kk; i++)
396 recoveredpystrs[i] = NULL;
397 fastblocks = PySequence_Fast(blocks, "First argument was not a sequence.");
400 fastblocknums = PySequence_Fast(blocknums, "Second argument was not a sequence.");
404 if (PySequence_Fast_GET_SIZE(fastblocks) != self->kk) {
405 PyErr_Format(py_fec_error, "Precondition violation: Wrong length -- first argument is required to contain exactly k blocks. len(first): %d, k: %d", PySequence_Fast_GET_SIZE(fastblocks), self->kk);
408 if (PySequence_Fast_GET_SIZE(fastblocknums) != self->kk) {
409 PyErr_Format(py_fec_error, "Precondition violation: Wrong length -- blocknums is required to contain exactly k blocks. len(blocknums): %d, k: %d", PySequence_Fast_GET_SIZE(fastblocknums), self->kk);
413 /* Construct a C array of gf*'s of the data and another of C ints of the blocknums. */
414 fastblocknumsitems = PySequence_Fast_ITEMS(fastblocknums);
415 if (!fastblocknumsitems)
417 fastblocksitems = PySequence_Fast_ITEMS(fastblocks);
418 if (!fastblocksitems)
421 for (i=0; i<self->kk; i++) {
422 if (!PyInt_Check(fastblocknumsitems[i])) {
423 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to contain int.");
426 tmpl = PyInt_AsLong(fastblocknumsitems[i]);
427 if (tmpl < 0 || tmpl > 255) {
428 PyErr_Format(py_fec_error, "Precondition violation: block nums can't be less than zero or greater than 255. %ld\n", tmpl);
431 cblocknums[i] = (unsigned)tmpl;
432 if (cblocknums[i] >= self->kk)
435 if (!PyObject_CheckReadBuffer(fastblocksitems[i])) {
436 PyErr_Format(py_fec_error, "Precondition violation: %u'th item is required to offer the single-segment read character buffer protocol, but it does not.\n", i);
439 if (PyObject_AsReadBuffer(fastblocksitems[i], (const void**)&(cblocks[i]), &sz))
441 if (oldsz != 0 && oldsz != sz) {
442 PyErr_Format(py_fec_error, "Precondition violation: Input blocks are required to be all the same length. length of one block was: %Zu, length of another block was: %Zu\n", oldsz, sz);
448 /* move src packets into position */
449 for (i=0; i<self->kk;) {
450 if (cblocknums[i] >= self->kk || cblocknums[i] == i)
453 /* put pkt in the right position. */
454 unsigned c = cblocknums[i];
456 SWAP (cblocknums[i], cblocknums[c], int);
457 SWAP (cblocks[i], cblocks[c], const gf*);
458 SWAP (fastblocksitems[i], fastblocksitems[c], PyObject*);
462 /* Allocate space for all of the recovered blocks. */
463 for (i=0; i<needtorecover; i++) {
464 recoveredpystrs[i] = PyString_FromStringAndSize(NULL, sz);
465 if (recoveredpystrs[i] == NULL)
467 recoveredcstrs[i] = (gf*)PyString_AsString(recoveredpystrs[i]);
468 if (recoveredcstrs[i] == NULL)
472 /* Decode any recovered blocks that are needed. */
473 fec_decode(self->fec_matrix, cblocks, recoveredcstrs, cblocknums, sz);
475 /* Wrap up both original primary blocks and decoded blocks into a Python list of Python strings. */
476 result = PyList_New(self->kk);
479 for (i=0; i<self->kk; i++) {
480 if (cblocknums[i] == i) {
481 /* Original primary block. */
482 Py_INCREF(fastblocksitems[i]);
483 if (PyList_SetItem(result, i, fastblocksitems[i]) == -1) {
484 Py_DECREF(fastblocksitems[i]);
488 /* Recovered block. */
489 if (PyList_SetItem(result, i, recoveredpystrs[nextrecoveredix]) == -1)
491 recoveredpystrs[nextrecoveredix] = NULL;
498 for (i=0; i<self->kk; i++)
499 Py_XDECREF(recoveredpystrs[i]);
500 Py_XDECREF(result); result = NULL;
502 Py_XDECREF(fastblocks); fastblocks=NULL;
503 Py_XDECREF(fastblocknums); fastblocknums=NULL;
508 Decoder_dealloc(Decoder * self) {
509 fec_free(self->fec_matrix);
510 self->ob_type->tp_free((PyObject*)self);
513 static PyMethodDef Decoder_methods[] = {
514 {"decode", (PyCFunction)Decoder_decode, METH_VARARGS, Decoder_decode__doc__},
518 static PyMemberDef Decoder_members[] = {
519 {"k", T_SHORT, offsetof(Encoder, kk), READONLY, "k"},
520 {"m", T_SHORT, offsetof(Encoder, mm), READONLY, "m"},
521 {NULL} /* Sentinel */
524 static PyTypeObject Decoder_type = {
525 PyObject_HEAD_INIT(NULL)
527 "_fec.Decoder", /*tp_name*/
528 sizeof(Decoder), /*tp_basicsize*/
530 (destructor)Decoder_dealloc, /*tp_dealloc*/
537 0, /*tp_as_sequence*/
545 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
546 Decoder__doc__, /* tp_doc */
549 0, /* tp_richcompare */
550 0, /* tp_weaklistoffset */
553 Decoder_methods, /* tp_methods */
554 Decoder_members, /* tp_members */
558 0, /* tp_descr_get */
559 0, /* tp_descr_set */
560 0, /* tp_dictoffset */
561 (initproc)Decoder_init, /* tp_init */
563 Decoder_new, /* tp_new */
566 static PyMethodDef fec_methods[] = {
570 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
571 #define PyMODINIT_FUNC void
576 PyObject *module_dict;
578 if (PyType_Ready(&Encoder_type) < 0)
580 if (PyType_Ready(&Decoder_type) < 0)
583 module = Py_InitModule3("_fec", fec_methods, fec__doc__);
587 Py_INCREF(&Encoder_type);
588 Py_INCREF(&Decoder_type);
590 PyModule_AddObject(module, "Encoder", (PyObject *)&Encoder_type);
591 PyModule_AddObject(module, "Decoder", (PyObject *)&Decoder_type);
593 module_dict = PyModule_GetDict(module);
594 py_fec_error = PyErr_NewException("_fec.Error", NULL, NULL);
595 PyDict_SetItemString(module_dict, "Error", py_fec_error);
599 * zfec -- fast forward error correction library with Python interface
601 * Copyright (C) 2007 Allmydata, Inc.
602 * Author: Zooko Wilcox-O'Hearn
604 * This file is part of zfec.
606 * See README.txt for licensing information.
610 * originally inspired by fecmodule.c by the Mnet Project, especially Myers
611 * Carpenter and Hauke Johannknecht