2 * zfec -- fast forward error correction library with Python interface
6 #include <structmember.h>
10 #if (PY_VERSION_HEX < 0x02050000)
11 typedef int Py_ssize_t;
18 static PyObject *py_fec_error;
20 static char fec__doc__[] = "\
21 FEC - Forward Error Correction \n\
24 static char Encoder__doc__[] = "\
25 Hold static encoder state (an in-memory table for matrix multiplication), and k and m parameters, and provide {encode()} method.\n\n\
26 @param k: the number of packets required for reconstruction \n\
27 @param m: the number of packets generated \n\
42 Encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwdict) {
45 self = (Encoder*)type->tp_alloc(type, 0);
49 self->fec_matrix = NULL;
52 return (PyObject *)self;
56 Encoder_init(Encoder *self, PyObject *args, PyObject *kwdict) {
57 static char *kwlist[] = {
63 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "ii:Encoder.__init__", kwlist, &ink, &inm))
67 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be greater than or equal to 1, but it was %d", ink);
71 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be greater than or equal to 1, but it was %d", inm);
75 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be less than or equal to 256, but it was %d", inm);
79 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be less than or equal to the second argument, but they were %d and %d respectively", ink, inm);
82 self->kk = (unsigned short)ink;
83 self->mm = (unsigned short)inm;
84 self->fec_matrix = fec_new(self->kk, self->mm);
89 static char Encoder_encode__doc__[] = "\
90 Encode data into m packets.\n\
92 @param inblocks: a sequence of k buffers of data to encode -- these are the k primary blocks, i.e. the input data split into k pieces (for best performance, make it a tuple instead of a list); All blocks are required to be the same length.\n\
93 @param desired_blocks_nums optional sequence of blocknums indicating which blocks to produce and return; If None, all m blocks will be returned (in order). (For best performance, make it a tuple instead of a list.)\n\
94 @returns: a list of buffers containing the requested blocks; Note that if any of the input blocks were 'primary blocks', i.e. their blocknum was < k, then the result sequence will contain a Python reference to the same Python object as was passed in. As long as the Python object in question is immutable (i.e. a string) then you don't have to think about this detail, but if it is mutable (i.e. an array), then you have to be aware that if you subsequently mutate the contents of that object then that will also change the contents of the sequence that was returned from this call to encode().\n\
98 Encoder_encode(Encoder *self, PyObject *args) {
100 PyObject* desired_blocks_nums = NULL; /* The blocknums of the blocks that should be returned. */
101 PyObject* result = NULL;
103 gf** check_blocks_produced = (gf**)alloca((self->mm - self->kk) * sizeof(PyObject*)); /* This is an upper bound -- we will actually use only num_check_blocks_produced of these elements (see below). */
104 PyObject** pystrs_produced = (PyObject**)alloca((self->mm - self->kk) * sizeof(PyObject*)); /* This is an upper bound -- we will actually use only num_check_blocks_produced of these elements (see below). */
105 unsigned num_check_blocks_produced = 0; /* The first num_check_blocks_produced elements of the check_blocks_produced array and of the pystrs_produced array will be used. */
106 const gf** incblocks = (const gf**)alloca(self->kk * sizeof(const gf*));
107 size_t num_desired_blocks;
108 PyObject* fast_desired_blocks_nums = NULL;
109 PyObject** fast_desired_blocks_nums_items;
110 size_t * c_desired_blocks_nums = (size_t*)alloca(self->mm * sizeof(size_t));
111 unsigned* c_desired_checkblocks_ids = (unsigned*)alloca((self->mm - self->kk) * sizeof(unsigned));
112 PyObject* fastinblocks = NULL;
113 PyObject** fastinblocksitems;
114 Py_ssize_t sz, oldsz = 0;
115 unsigned char check_block_index = 0; /* index into the check_blocks_produced and (parallel) pystrs_produced arrays */
117 if (!PyArg_ParseTuple(args, "O|O:Encoder.encode", &inblocks, &desired_blocks_nums))
120 for (size_t i = 0; i < self->mm - self->kk; i++)
121 pystrs_produced[i] = NULL;
123 if (desired_blocks_nums) {
124 fast_desired_blocks_nums = PySequence_Fast(desired_blocks_nums, "Second argument (optional) was not a sequence.");
126 if (!fast_desired_blocks_nums)
129 num_desired_blocks = PySequence_Fast_GET_SIZE(fast_desired_blocks_nums);
130 fast_desired_blocks_nums_items = PySequence_Fast_ITEMS(fast_desired_blocks_nums);
132 for (size_t i = 0; i < num_desired_blocks; i++) {
133 if (!PyInt_Check(fast_desired_blocks_nums_items[i])) {
134 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to contain int.");
137 c_desired_blocks_nums[i] = PyInt_AsLong(fast_desired_blocks_nums_items[i]);
138 if (c_desired_blocks_nums[i] >= self->kk)
139 num_check_blocks_produced++;
142 num_desired_blocks = self->mm;
143 for (size_t i = 0; i<num_desired_blocks; i++)
144 c_desired_blocks_nums[i] = i;
145 num_check_blocks_produced = self->mm - self->kk;
148 fastinblocks = PySequence_Fast(inblocks, "First argument was not a sequence.");
152 if (PySequence_Fast_GET_SIZE(fastinblocks) != self->kk) {
153 PyErr_Format(py_fec_error, "Precondition violation: Wrong length -- first argument (the sequence of input blocks) is required to contain exactly k blocks. len(first): %zu, k: %d", PySequence_Fast_GET_SIZE(fastinblocks), self->kk);
157 /* Construct a C array of gf*'s of the input data. */
158 fastinblocksitems = PySequence_Fast_ITEMS(fastinblocks);
159 if (!fastinblocksitems)
162 for (size_t i = 0; i < self->kk; i++) {
163 if (!PyObject_CheckReadBuffer(fastinblocksitems[i])) {
164 PyErr_Format(py_fec_error, "Precondition violation: %zu'th item is required to offer the single-segment read character buffer protocol, but it does not.", i);
167 if (PyObject_AsReadBuffer(fastinblocksitems[i], (const void**)&(incblocks[i]), &sz))
169 if (oldsz != 0 && oldsz != sz) {
170 PyErr_Format(py_fec_error, "Precondition violation: Input blocks are required to be all the same length. length of one block was: %zu, length of another block was: %zu", oldsz, sz);
176 /* Allocate space for all of the check blocks. */
178 for (size_t i = 0; i < num_desired_blocks; i++) {
179 if (c_desired_blocks_nums[i] >= self->kk) {
180 c_desired_checkblocks_ids[check_block_index] = c_desired_blocks_nums[i];
181 pystrs_produced[check_block_index] = PyString_FromStringAndSize(NULL, sz);
182 if (pystrs_produced[check_block_index] == NULL)
184 check_blocks_produced[check_block_index] = (gf*)PyString_AsString(pystrs_produced[check_block_index]);
185 if (check_blocks_produced[check_block_index] == NULL)
190 assert (check_block_index == num_check_blocks_produced);
192 /* Encode any check blocks that are needed. */
193 fec_encode(self->fec_matrix, incblocks, check_blocks_produced, c_desired_checkblocks_ids, num_check_blocks_produced, sz);
195 /* Wrap all requested blocks up into a Python list of Python strings. */
196 result = PyList_New(num_desired_blocks);
199 check_block_index = 0;
200 for (size_t i = 0; i < num_desired_blocks; i++) {
201 if (c_desired_blocks_nums[i] < self->kk) {
202 Py_INCREF(fastinblocksitems[c_desired_blocks_nums[i]]);
203 if (PyList_SetItem(result, i, fastinblocksitems[c_desired_blocks_nums[i]]) == -1) {
204 Py_DECREF(fastinblocksitems[c_desired_blocks_nums[i]]);
208 if (PyList_SetItem(result, i, pystrs_produced[check_block_index]) == -1)
210 pystrs_produced[check_block_index] = NULL;
217 for (size_t i = 0; i < num_check_blocks_produced; i++)
218 Py_XDECREF(pystrs_produced[i]);
219 Py_XDECREF(result); result = NULL;
221 Py_XDECREF(fastinblocks); fastinblocks=NULL;
222 Py_XDECREF(fast_desired_blocks_nums); fast_desired_blocks_nums=NULL;
227 Encoder_dealloc(Encoder * self) {
228 if (self->fec_matrix)
229 fec_free(self->fec_matrix);
230 self->ob_type->tp_free((PyObject*)self);
233 static PyMethodDef Encoder_methods[] = {
234 {"encode", (PyCFunction)Encoder_encode, METH_VARARGS, Encoder_encode__doc__},
238 static PyMemberDef Encoder_members[] = {
239 {"k", T_SHORT, offsetof(Encoder, kk), READONLY, "k"},
240 {"m", T_SHORT, offsetof(Encoder, mm), READONLY, "m"},
241 {NULL} /* Sentinel */
244 static PyTypeObject Encoder_type = {
245 PyObject_HEAD_INIT(NULL)
247 "_fec.Encoder", /*tp_name*/
248 sizeof(Encoder), /*tp_basicsize*/
250 (destructor)Encoder_dealloc, /*tp_dealloc*/
257 0, /*tp_as_sequence*/
265 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
266 Encoder__doc__, /* tp_doc */
269 0, /* tp_richcompare */
270 0, /* tp_weaklistoffset */
273 Encoder_methods, /* tp_methods */
274 Encoder_members, /* tp_members */
278 0, /* tp_descr_get */
279 0, /* tp_descr_set */
280 0, /* tp_dictoffset */
281 (initproc)Encoder_init, /* tp_init */
283 Encoder_new, /* tp_new */
286 static char Decoder__doc__[] = "\
287 Hold static decoder state (an in-memory table for matrix multiplication), and k and m parameters, and provide {decode()} method.\n\n\
288 @param k: the number of packets required for reconstruction \n\
289 @param m: the number of packets generated \n\
304 Decoder_new(PyTypeObject *type, PyObject *args, PyObject *kwdict) {
307 self = (Decoder*)type->tp_alloc(type, 0);
311 self->fec_matrix = NULL;
314 return (PyObject *)self;
318 Decoder_init(Encoder *self, PyObject *args, PyObject *kwdict) {
319 static char *kwlist[] = {
326 if (!PyArg_ParseTupleAndKeywords(args, kwdict, "ii:Decoder.__init__", kwlist, &ink, &inm))
330 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be greater than or equal to 1, but it was %d", ink);
334 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be greater than or equal to 1, but it was %d", inm);
338 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to be less than or equal to 256, but it was %d", inm);
342 PyErr_Format(py_fec_error, "Precondition violation: first argument is required to be less than or equal to the second argument, but they were %d and %d respectively", ink, inm);
345 self->kk = (unsigned short)ink;
346 self->mm = (unsigned short)inm;
347 self->fec_matrix = fec_new(self->kk, self->mm);
352 #define SWAP(a,b,t) {t tmp; tmp=a; a=b; b=tmp;}
354 static char Decoder_decode__doc__[] = "\
355 Decode a list blocks into a list of segments.\n\
356 @param blocks a sequence of buffers containing block data (for best performance, make it a tuple instead of a list)\n\
357 @param blocknums a sequence of integers of the blocknum for each block in blocks (for best performance, make it a tuple instead of a list)\n\
359 @return a list of strings containing the segment data (i.e. ''.join(retval) yields a string containing the decoded data)\n\
363 Decoder_decode(Decoder *self, PyObject *args) {
364 PyObject*restrict blocks;
365 PyObject*restrict blocknums;
366 PyObject* result = NULL;
368 const gf**restrict cblocks = (const gf**restrict)alloca(self->kk * sizeof(const gf*));
369 unsigned* cblocknums = (unsigned*)alloca(self->kk * sizeof(unsigned));
370 gf**restrict recoveredcstrs = (gf**)alloca(self->kk * sizeof(gf*)); /* self->kk is actually an upper bound -- we probably won't need all of this space. */
371 PyObject**restrict recoveredpystrs = (PyObject**restrict)alloca(self->kk * sizeof(PyObject*)); /* self->kk is actually an upper bound -- we probably won't need all of this space. */
373 PyObject*restrict fastblocknums = NULL;
374 PyObject*restrict fastblocks;
375 unsigned needtorecover=0;
376 PyObject** fastblocksitems;
377 PyObject** fastblocknumsitems;
378 Py_ssize_t sz, oldsz = 0;
380 unsigned nextrecoveredix=0;
382 if (!PyArg_ParseTuple(args, "OO:Decoder.decode", &blocks, &blocknums))
385 for (i=0; i<self->kk; i++)
386 recoveredpystrs[i] = NULL;
387 fastblocks = PySequence_Fast(blocks, "First argument was not a sequence.");
390 fastblocknums = PySequence_Fast(blocknums, "Second argument was not a sequence.");
394 if (PySequence_Fast_GET_SIZE(fastblocks) != self->kk) {
395 PyErr_Format(py_fec_error, "Precondition violation: Wrong length -- first argument is required to contain exactly k blocks. len(first): %zu, k: %d", PySequence_Fast_GET_SIZE(fastblocks), self->kk);
398 if (PySequence_Fast_GET_SIZE(fastblocknums) != self->kk) {
399 PyErr_Format(py_fec_error, "Precondition violation: Wrong length -- blocknums is required to contain exactly k blocks. len(blocknums): %zu, k: %d", PySequence_Fast_GET_SIZE(fastblocknums), self->kk);
403 /* Construct a C array of gf*'s of the data and another of C ints of the blocknums. */
404 fastblocknumsitems = PySequence_Fast_ITEMS(fastblocknums);
405 if (!fastblocknumsitems)
407 fastblocksitems = PySequence_Fast_ITEMS(fastblocks);
408 if (!fastblocksitems)
411 for (i=0; i<self->kk; i++) {
412 if (!PyInt_Check(fastblocknumsitems[i])) {
413 PyErr_Format(py_fec_error, "Precondition violation: second argument is required to contain int.");
416 tmpl = PyInt_AsLong(fastblocknumsitems[i]);
417 if (tmpl < 0 || tmpl > 255) {
418 PyErr_Format(py_fec_error, "Precondition violation: block nums can't be less than zero or greater than 255. %ld\n", tmpl);
421 cblocknums[i] = (unsigned)tmpl;
422 if (cblocknums[i] >= self->kk)
425 if (!PyObject_CheckReadBuffer(fastblocksitems[i])) {
426 PyErr_Format(py_fec_error, "Precondition violation: %u'th item is required to offer the single-segment read character buffer protocol, but it does not.\n", i);
429 if (PyObject_AsReadBuffer(fastblocksitems[i], (const void**)&(cblocks[i]), &sz))
431 if (oldsz != 0 && oldsz != sz) {
432 PyErr_Format(py_fec_error, "Precondition violation: Input blocks are required to be all the same length. length of one block was: %zu, length of another block was: %zu\n", oldsz, sz);
438 /* Move src packets into position. At the end of this loop we want the i'th
439 element of the arrays to be the block with block number i, if that block
440 is among our inputs. */
441 for (i=0; i<self->kk;) {
442 if (cblocknums[i] >= self->kk || cblocknums[i] == i)
445 /* put pkt in the right position. */
446 unsigned c = cblocknums[i];
448 SWAP (cblocknums[i], cblocknums[c], int);
449 SWAP (cblocks[i], cblocks[c], const gf*);
450 SWAP (fastblocksitems[i], fastblocksitems[c], PyObject*);
454 /* Allocate space for all of the recovered blocks. */
455 for (i=0; i<needtorecover; i++) {
456 recoveredpystrs[i] = PyString_FromStringAndSize(NULL, sz);
457 if (recoveredpystrs[i] == NULL)
459 recoveredcstrs[i] = (gf*)PyString_AsString(recoveredpystrs[i]);
460 if (recoveredcstrs[i] == NULL)
464 /* Decode any recovered blocks that are needed. */
465 fec_decode(self->fec_matrix, cblocks, recoveredcstrs, cblocknums, sz);
467 /* Wrap up both original primary blocks and decoded blocks into a Python list of Python strings. */
468 result = PyList_New(self->kk);
471 for (i=0; i<self->kk; i++) {
472 if (cblocknums[i] == i) {
473 /* Original primary block. */
474 Py_INCREF(fastblocksitems[i]);
475 if (PyList_SetItem(result, i, fastblocksitems[i]) == -1) {
476 Py_DECREF(fastblocksitems[i]);
480 /* Recovered block. */
481 if (PyList_SetItem(result, i, recoveredpystrs[nextrecoveredix]) == -1)
483 recoveredpystrs[nextrecoveredix] = NULL;
490 for (i=0; i<self->kk; i++)
491 Py_XDECREF(recoveredpystrs[i]);
492 Py_XDECREF(result); result = NULL;
494 Py_XDECREF(fastblocks); fastblocks=NULL;
495 Py_XDECREF(fastblocknums); fastblocknums=NULL;
500 Decoder_dealloc(Decoder * self) {
501 if (self->fec_matrix)
502 fec_free(self->fec_matrix);
503 self->ob_type->tp_free((PyObject*)self);
506 static PyMethodDef Decoder_methods[] = {
507 {"decode", (PyCFunction)Decoder_decode, METH_VARARGS, Decoder_decode__doc__},
511 static PyMemberDef Decoder_members[] = {
512 {"k", T_SHORT, offsetof(Encoder, kk), READONLY, "k"},
513 {"m", T_SHORT, offsetof(Encoder, mm), READONLY, "m"},
514 {NULL} /* Sentinel */
517 static PyTypeObject Decoder_type = {
518 PyObject_HEAD_INIT(NULL)
520 "_fec.Decoder", /*tp_name*/
521 sizeof(Decoder), /*tp_basicsize*/
523 (destructor)Decoder_dealloc, /*tp_dealloc*/
530 0, /*tp_as_sequence*/
538 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
539 Decoder__doc__, /* tp_doc */
542 0, /* tp_richcompare */
543 0, /* tp_weaklistoffset */
546 Decoder_methods, /* tp_methods */
547 Decoder_members, /* tp_members */
551 0, /* tp_descr_get */
552 0, /* tp_descr_set */
553 0, /* tp_dictoffset */
554 (initproc)Decoder_init, /* tp_init */
556 Decoder_new, /* tp_new */
561 _hexwrite(unsigned char*s, size_t l) {
563 for (i = 0; i < l; i++)
564 printf("%.2x", s[i]);
569 test_from_agl(PyObject* self, PyObject* args) {
570 unsigned char b0c[8], b1c[8];
571 unsigned char b0[8], b1[8], b2[8], b3[8], b4[8];
573 const unsigned char *blocks[3] = {b0, b1, b2};
574 unsigned char *outblocks[2] = {b3, b4};
575 unsigned block_nums[] = {3, 4};
577 fec_t *const fec = fec_new(3, 5);
579 const unsigned char *inpkts[] = {b3, b4, b2};
580 unsigned char *outpkts[] = {b0, b1};
581 unsigned indexes[] = {3, 4, 2};
587 /*printf("_from_c before encoding:\n");
588 printf("b0: "); _hexwrite(b0, 8); printf(", ");
589 printf("b1: "); _hexwrite(b1, 8); printf(", ");
590 printf("b2: "); _hexwrite(b2, 8); printf(", ");
593 fec_encode(fec, blocks, outblocks, block_nums, 2, 8);
595 /*printf("after encoding:\n");
596 printf("b3: "); _hexwrite(b3, 8); printf(", ");
597 printf("b4: "); _hexwrite(b4, 8); printf(", ");
600 memcpy(b0c, b0, 8); memcpy(b1c, b1, 8);
602 fec_decode(fec, inpkts, outpkts, indexes, 8);
604 /*printf("after decoding:\n");
605 printf("b0: "); _hexwrite(b0, 8); printf(", ");
606 printf("b1: "); _hexwrite(b1, 8);
609 if ((memcmp(b0, b0c,8) == 0) && (memcmp(b1, b1c,8) == 0))
615 static PyMethodDef fec_functions[] = {
616 {"test_from_agl", test_from_agl, METH_NOARGS, NULL},
620 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
621 #define PyMODINIT_FUNC void
626 PyObject *module_dict;
628 if (PyType_Ready(&Encoder_type) < 0)
630 if (PyType_Ready(&Decoder_type) < 0)
633 module = Py_InitModule3("_fec", fec_functions, fec__doc__);
637 Py_INCREF(&Encoder_type);
638 Py_INCREF(&Decoder_type);
640 PyModule_AddObject(module, "Encoder", (PyObject *)&Encoder_type);
641 PyModule_AddObject(module, "Decoder", (PyObject *)&Decoder_type);
643 module_dict = PyModule_GetDict(module);
644 py_fec_error = PyErr_NewException("_fec.Error", NULL, NULL);
645 PyDict_SetItemString(module_dict, "Error", py_fec_error);
649 * originally inspired by fecmodule.c by the Mnet Project, especially Myers
650 * Carpenter and Hauke Johannknecht