Tiny, modular, drop-in, library of some most commonly used utility methods for C (embedded) applications. Intended to be used as a git-submodule inside your projects to kickstart development. See https://c-utils.gotomain.io for more details.
Home Page: https://gotomain.io
License: Apache License 2.0
if enough pushes and pops are done, push_count in circular-generic-buffer.c will eventually overflow, causing errors.
Attached test program will show the issue, it eventually gets to
* push_count = 65535, pop_count = 65527
popped 65528
* push_count = 65535, pop_count = 65528
pushed 65536
* push_count = 0, pop_count = 65528
CB is empty
Test Passed, took 0 iteration
* push_count = 0, pop_count = 65528
recommend using head and tail like circular_buffer.c, and add a count field to detect buffer full.
Patch:
--- a/circular-generic-buffer.h
+++ b/circular-generic-buffer.h
@@ -32,8 +32,9 @@
typedef struct {
void * const buffer;
- uint16_t push_count;
- uint16_t pop_count;
+ uint16_t i_head;
+ uint16_t i_tail;
+ uint16_t count;
uint16_t const size;
uint16_t const element_size;
} circ_gbuf_t;
@@ -42,23 +43,24 @@ typedef struct {
t x ## _circ_gbuf_data[s]; \
circ_gbuf_t x = { \
.buffer = x ## _circ_gbuf_data, \
- .push_count = 0, \
- .pop_count = 0, \
- .size = s , \
+ .i_head = 0, \
+ .i_tail = 0, \
+ .count = 0, \
+ .size = s, \
.element_size = sizeof(t) \
};
--- a/circular-generic-buffer.c
+++ b/circular-generic-buffer.c
@@ -28,7 +28,7 @@
#include <string.h>
-#include "circular-generic-buffer-count.h"
+#include "circular-generic-buffer.h"
static inline void zero_pad(char *s,int len)
{
@@ -41,11 +41,10 @@ int circ_gbuf_pop(circ_gbuf_t *circ_buf, void *elem, int read_only)
{
char *tail;
- if (circ_buf->pop_count >= circ_buf->push_count)
+ if (circ_buf->count == 0)
return -1; // quit with an error
- tail = circ_buf->buffer + ((circ_buf->pop_count%circ_buf->size)
- * circ_buf->element_size);
+ tail = circ_buf->buffer + (circ_buf->i_tail * circ_buf->element_size);
if (elem)
memcpy(elem, tail, circ_buf->element_size);
@@ -53,7 +52,8 @@ int circ_gbuf_pop(circ_gbuf_t *circ_buf, void *elem, int read_only)
if (!read_only) {
*tail=0;
zero_pad(tail , circ_buf->element_size);
- circ_buf->pop_count++;
+ circ_buf->i_tail = (circ_buf->i_tail + 1) % circ_buf->size;
+ circ_buf->count--;
}
return 0;
}
@@ -62,21 +62,32 @@ int circ_gbuf_push(circ_gbuf_t *circ_buf, void *elem)
{
char *head;
- if ((circ_buf->push_count - circ_buf->pop_count) >= circ_buf->size)
+ if (circ_buf->count >= circ_buf->size)
return -1;
- head = circ_buf->buffer + ( (circ_buf->push_count % circ_buf->size)
- * circ_buf->element_size );
+ head = circ_buf->buffer + (circ_buf->i_head * circ_buf->element_size);
memcpy(head, elem, circ_buf->element_size);
- circ_buf->push_count++;
+ circ_buf->i_head = (circ_buf->i_head + 1) % circ_buf->size;
+ circ_buf->count++;
return 0;
}
int circ_gbuf_free_space(circ_gbuf_t *circBuf)
{
- return circBuf->size - (circBuf->pop_count - circBuf->push_count);
+ return circBuf->size - circBuf->count;
}
The function circ_gbuf_free_space uses:
uint16_t push_count;
uint16_t pop_count;
as follows:
int circ_gbuf_free_space(circ_gbuf_t *circ_buf)
{
int total;
total = circ_buf->push_count - circ_buf->pop_count;
if (total < 0)
total += (2 * circ_buf->size);
return circ_buf->size - total;
}
It is easy to prove that circ_gbuf_free_space is not a thread safe function.
Since pop and push use exactly same calculations:
int circ_gbuf_pop(circ_gbuf_t *circ_buf, void *elem, int read_only)
{
...
total = circ_buf->push_count - circ_buf->pop_count;
if (total < 0)
total += (2 * circ_buf->size);
...
}
int circ_gbuf_push(circ_gbuf_t *circ_buf, void *elem)
{
...
total = circ_buf->push_count - circ_buf->pop_count;
if (total < 0)
total += (2 * circ_buf->size);
...
}
then they are also not thread safe.
Test program which proves that even with one producer and one consumer calculations will go wrong:
#include <unistd.h> /* sleep */
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "circular-generic-buffer.h"
struct my_struct {
int a;
char b;
double c;
};
#define BUF_SIZE 100
#define RUN_THREAD 1
CIRC_GBUF_DEF(struct my_struct, my_circ_buf, BUF_SIZE)
#if RUN_THREAD
#include <sched.h>
#include <pthread.h> /* phread_create */
void * push(void * arg)
{
// UNUSED(arg);
int i;
struct my_struct test;
int nr;
int free_space;
printf("push tasks started\n");
//2. One task is pushing BUF_SIZE/2 times
for (i=0; i<BUF_SIZE/2; i++) {
test.a = i;
nr = circ_gbuf_free_space(&my_circ_buf);
/* some calculations */
usleep(i);
/* some calculations */
free_space = circ_gbuf_free_space(&my_circ_buf);
if (nr!=free_space)
printf("+iter=%d pop_count=%d push_count=%d free_space=%d free_space=%d\n", i,
my_circ_buf.pop_count, my_circ_buf.push_count, free_space, nr);
if ( circ_gbuf_push(&my_circ_buf, &test) ) {
printf("*2. Out of space in CB at %d\n", i);
return NULL;
}
}
return NULL;
}
void * pop(void * arg)
{
// UNUSED(arg);
int i;
struct my_struct test;
int nr;
int free_space;
printf("pop tasks started\n");
//2. One task is pushing BUF_SIZE/2 times
for (i=0; i<BUF_SIZE/2; i++) {
if ( circ_gbuf_pop( &my_circ_buf, &test, 0) /*CIRC_GBUF_POP(my_circ_buf, &a[10 + i])*/ ) {
printf("*3. CB is empty at %d\n", i);
return NULL;
}
nr = circ_gbuf_free_space(&my_circ_buf);
/* some calculations */
if( i%2) usleep(i);
/* some calculations */
free_space = circ_gbuf_free_space(&my_circ_buf);
if (nr!=free_space)
printf("-iter=%d pop_count=%d push_count=%d free_space=%d free_space=%d\n", i,
my_circ_buf.pop_count, my_circ_buf.push_count, free_space, nr);
}
return NULL;
}
void * observer(void * arg){
int i;
struct my_struct test;
int nr;
int free_space;
printf("observer tasks started\n");
for (i=0; i<BUF_SIZE; i++) {
if( i%2) usleep(i);
}
return NULL;
}
#endif
int main()
{
int i;
int j;
struct my_struct a[20];
#if RUN_THREAD
/*
pthread_mutex_t mutex;
pthread_cond_t cond;
*/
pthread_t task1;
pthread_t task2;
pthread_t task3;
#endif
struct my_struct test;
test.a= 0;
test.b= 'S';
test.c = 12345.678;
int nr,free_space;
srand(time(NULL));
//1. fill the buffer in half
for (i=0; i<BUF_SIZE/2; i++) {
test.a = i;
if ( circ_gbuf_push(&my_circ_buf, &test) ) {
printf("1. Out of space in CB at %d\n", i);
return -1;
}
}
nr = circ_gbuf_free_space(&my_circ_buf);
/* some calculations */
usleep(200);
/* some calculations */
free_space = circ_gbuf_free_space(&my_circ_buf);
printf("start pop_count=%d push_count=%d free_space=%d free_space=%d\n",
my_circ_buf.pop_count, my_circ_buf.push_count, free_space, nr);
#if 0
//2. One task is pushing BUF_SIZE/2 times
for (i=0; i<BUF_SIZE/2; i++) {
test.a = i;
if ( circ_gbuf_push(&my_circ_buf, &test) ) {
printf("2. Out of space in CB at %d\n", i);
return -1;
}
}
//3. One task is poping BUF_SIZE/2 times
for (i=0; i<BUF_SIZE/2; i++) {
if ( circ_gbuf_pop( &my_circ_buf, &test, 0) /*CIRC_GBUF_POP(my_circ_buf, &a[10 + i])*/ ) {
printf("3. CB is empty at %d\n", i);
return -1;
}
}
printf("Test Passed .pop_count=%d .push_count=%d nr_of_elem=%d\n",
my_circ_buf.pop_count, my_circ_buf.push_count,circ_gbuf_free_space(&my_circ_buf));
#endif
#if RUN_THREAD
printf("ok1!\n");
if( pthread_create(&task1, NULL, push, NULL) != 0)
{
printf("problem:task1\n");
}
printf("ok2!\n");
printf("ok3!\n");
#if 0
if( pthread_create(&task3, NULL, observer, NULL) != 0)
{
printf("problem:task3\n");
}
#endif
printf("ok4!\n");
printf("ok5!\n");
if( pthread_create(&task2, NULL, pop, NULL) != 0)
{
printf("problem:task2\n");
}
printf("ok6!\n");
pthread_join( task1, NULL);
#if 0
pthread_join( task3, NULL);
#endif
pthread_join( task2, NULL);
printf("Test results .pop_count=%d .push_count=%d free_space=%d\n",
my_circ_buf.pop_count, my_circ_buf.push_count,circ_gbuf_free_space(&my_circ_buf));
#endif
return 0;
}
Results:
start pop_count=0 push_count=50 free_space=50 free_space=50
ok1!
ok2!
ok3!
ok4!
ok5!
ok6!
pop tasks started
push tasks started
+iter=0 pop_count=4 push_count=50 free_space=54 free_space=52
-iter=3 pop_count=4 push_count=51 free_space=53 free_space=54
+iter=1 pop_count=6 push_count=51 free_space=55 free_space=53
-iter=5 pop_count=6 push_count=52 free_space=54 free_space=55
+iter=2 pop_count=8 push_count=52 free_space=56 free_space=54
-iter=7 pop_count=8 push_count=53 free_space=55 free_space=56
+iter=3 pop_count=10 push_count=53 free_space=57 free_space=55
-iter=9 pop_count=10 push_count=54 free_space=56 free_space=57
+iter=4 pop_count=12 push_count=54 free_space=58 free_space=56
-iter=11 pop_count=12 push_count=55 free_space=57 free_space=58
+iter=5 pop_count=14 push_count=55 free_space=59 free_space=57
-iter=13 pop_count=14 push_count=56 free_space=58 free_space=59
+iter=6 pop_count=16 push_count=56 free_space=60 free_space=58
-iter=15 pop_count=16 push_count=57 free_space=59 free_space=60
+iter=7 pop_count=18 push_count=57 free_space=61 free_space=59
-iter=17 pop_count=18 push_count=58 free_space=60 free_space=61
+iter=8 pop_count=20 push_count=58 free_space=62 free_space=60
-iter=19 pop_count=20 push_count=59 free_space=61 free_space=62
+iter=9 pop_count=22 push_count=59 free_space=63 free_space=61
-iter=21 pop_count=22 push_count=60 free_space=62 free_space=63
+iter=11 pop_count=26 push_count=61 free_space=65 free_space=61
-iter=25 pop_count=26 push_count=62 free_space=64 free_space=65
+iter=12 pop_count=28 push_count=62 free_space=66 free_space=64
-iter=27 pop_count=28 push_count=63 free_space=65 free_space=66
+iter=13 pop_count=30 push_count=63 free_space=67 free_space=65
-iter=29 pop_count=30 push_count=64 free_space=66 free_space=67
+iter=16 pop_count=32 push_count=66 free_space=66 free_space=64
-iter=31 pop_count=32 push_count=68 free_space=64 free_space=66
+iter=18 pop_count=34 push_count=68 free_space=66 free_space=64
-iter=33 pop_count=34 push_count=69 free_space=65 free_space=66
+iter=19 pop_count=36 push_count=69 free_space=67 free_space=65
-iter=35 pop_count=36 push_count=70 free_space=66 free_space=67
+iter=20 pop_count=38 push_count=70 free_space=68 free_space=66
-iter=37 pop_count=38 push_count=71 free_space=67 free_space=68
+iter=21 pop_count=40 push_count=71 free_space=69 free_space=67
-iter=39 pop_count=40 push_count=72 free_space=68 free_space=69
+iter=22 pop_count=42 push_count=72 free_space=70 free_space=68
-iter=41 pop_count=42 push_count=73 free_space=69 free_space=70
+iter=23 pop_count=44 push_count=73 free_space=71 free_space=69
-iter=43 pop_count=44 push_count=75 free_space=69 free_space=71
+iter=25 pop_count=46 push_count=75 free_space=71 free_space=69
-iter=45 pop_count=46 push_count=76 free_space=70 free_space=71
+iter=26 pop_count=48 push_count=76 free_space=72 free_space=70
-iter=47 pop_count=48 push_count=78 free_space=70 free_space=72
+iter=28 pop_count=50 push_count=78 free_space=72 free_space=70
-iter=49 pop_count=50 push_count=79 free_space=71 free_space=72
Test results .pop_count=50 .push_count=100 free_space=50
calculation should be return circBuf->size - (circBuf->push_count - circBuf->pop_count); ?
inline functions in CIRC_GBUF_DEF macro in circular-generic-buffer.h are not reliably inlined by gcc.
gcc -Wall -DC_UTILS_TESTING circular-generic-buffer.c
/tmp/cc9km6O1.o: In function `main':
circular-generic-buffer.c:(.text+0x281): undefined reference to `my_circ_buf_push_refd'
circular-generic-buffer.c:(.text+0x2f6): undefined reference to `my_circ_buf_pop_refd'
collect2: error: ld returned 1 exit status
recommends making the functions static inline. This fixes the issue.
The function:
int circ_gbuf_pop(circ_gbuf_t *circ_buf, void *elem, int read_only)
contains call to:
zero_pad(tail, circ_buf->element_size);
I am not sure if this function is needed. For speed improvement it could be removed. Am I wrong?
It seems that the description for CIRC_GBUF_PEEK and CIRC_GBUF_POP is interchanged. The functionality for PEEK (not removing the element) and POP (element is removed) is fine.
There seems to be an issue , perhaps with the preprocessor directives(?), which prevents compilation when the type specified is not a simple byte array.
Here is the sample code, and it running against GCC.
https://godbolt.org/z/fTsfaenfs
For reference :
#include <https://raw.githubusercontent.com/goToMain/c-utils/master/include/utils/circbuf.h>
#include <stdio.h>
typedef struct {
uint8_t can_header;
uint8_t data[8];
uint8_t errors;
uint8_t counter;
} can_rx_frame_t;
CIRCBUF_DEF(can_rx_frame_t, my_new_buf, 32);
int your_application()
{
can_rx_frame_t in_frame;
can_rx_frame_t out_frame;
in_frame.can_header = 1; // send a message
out_frame.can_header = 0; // overwrite this
if (my_new_buf_push(&my_new_buf, &in_frame)) {
printf("Out of space in CB\n");
return -1;
}
if (my_new_buf_pop(&my_new_buf, &out_frame)) {
printf("CB is empty\n");
return -1;
}
// here is the data
printf("Push: 0x%x\n", in_frame.can_header);
printf("Pop: 0x%x\n", out_frame.can_header);
return 0;
}
int main() {
your_application();
}
Warnings (errors really) are :
<source>: In function 'your_application':
<source>:23:9: warning: implicit declaration of function 'my_new_buf_push'; did you mean 'my_new_buf_push_refd'? [-Wimplicit-function-declaration]
23 | if (my_new_buf_push(&my_new_buf, &in_frame)) {
| ^~~~~~~~~~~~~~~
| my_new_buf_push_refd
<source>:28:9: warning: implicit declaration of function 'my_new_buf_pop'; did you mean 'my_new_buf_pop_refd'? [-Wimplicit-function-declaration]
28 | if (my_new_buf_pop(&my_new_buf, &out_frame)) {
| ^~~~~~~~~~~~~~
| my_new_buf_pop_refd
I can't get it to work using the function call my_new_buf_push_refd() either.
<source>: In function 'your_application':
<source>:23:30: warning: passing argument 1 of 'my_new_buf_push_refd' from incompatible pointer type [-Wincompatible-pointer-types]
23 | if (my_new_buf_push_refd(&my_new_buf, &in_frame)) {
| ^~~~~~~~~~~
| |
| circbuf_t *
In file included from <source>:1:
/app/raw.githubusercontent.com/goToMain/c-utils/master/include/utils/circbuf.h:53:37: note: expected 'can_rx_frame_t *' but argument is of type 'circbuf_t *'
53 | int buf ## _push_refd(type *pt) \
<source>:12:1: note: in expansion of macro 'CIRCBUF_DEF'
12 | CIRCBUF_DEF(can_rx_frame_t, my_new_buf, 32);
| ^~~~~~~~~~~
<source>:23:9: error: too many arguments to function 'my_new_buf_push_refd'
23 | if (my_new_buf_push_refd(&my_new_buf, &in_frame)) {
| ^~~~~~~~~~~~~~~~~~~~
<source>:12:29: note: declared here
12 | CIRCBUF_DEF(can_rx_frame_t, my_new_buf, 32);
| ^~~~~~~~~~
/app/raw.githubusercontent.com/goToMain/c-utils/master/include/utils/circbuf.h:53:13: note: in definition of macro 'CIRCBUF_DEF'
53 | int buf ## _push_refd(type *pt) \
| ^~~
<source>:28:29: warning: passing argument 1 of 'my_new_buf_pop_refd' from incompatible pointer type [-Wincompatible-pointer-types]
28 | if (my_new_buf_pop_refd(&my_new_buf, &out_frame)) {
| ^~~~~~~~~~~
| |
| circbuf_t *
/app/raw.githubusercontent.com/goToMain/c-utils/master/include/utils/circbuf.h:57:36: note: expected 'can_rx_frame_t *' but argument is of type 'circbuf_t *'
57 | int buf ## _pop_refd(type *pt) \
<source>:12:1: note: in expansion of macro 'CIRCBUF_DEF'
12 | CIRCBUF_DEF(can_rx_frame_t, my_new_buf, 32);
| ^~~~~~~~~~~
<source>:28:9: error: too many arguments to function 'my_new_buf_pop_refd'
28 | if (my_new_buf_pop_refd(&my_new_buf, &out_frame)) {
| ^~~~~~~~~~~~~~~~~~~
<source>:12:29: note: declared here
12 | CIRCBUF_DEF(can_rx_frame_t, my_new_buf, 32);
| ^~~~~~~~~~
/app/raw.githubusercontent.com/goToMain/c-utils/master/include/utils/circbuf.h:57:13: note: in definition of macro 'CIRCBUF_DEF'
57 | int buf ## _pop_refd(type *pt) \
| ^~~
Compiler returned: 1
Can you see what i may be doing wrong here ?
#include "circular-generic-buffer-count.h"
in circular-generic-buffer-count.c fails.
I have modified the original test to compare retrieved .b and .c elements of the structure.
These comparisons have been omitted in the original test.
This is test program:
int test2()
{
struct my_struct 1{
int a;
char b;
float c;
};
int i;
int nr_of_iter = 10;
struct my_struct1 a[2*nr_of_iter+2];
CIRC_GBUF_DEF(struct my_struct1, my_circ_buf, nr_of_iter+2);
srand(time(NULL));
for (i=0; i<nr_of_iter; i++) {
a[i].a = rand();
a[i].b = rand();
a[i].c = (float) rand();
if (CIRC_GBUF_PUSH(my_circ_buf, &a[i])) {
printf("Out of space in CB at %d\n", i);
return -1;
}
}
for (i=0; i<nr_of_iter; i++)
{
if (CIRC_GBUF_POP(my_circ_buf, &a[nr_of_iter + i])) {
printf("CB is empty at %d\n", i);
//return -1;
}
if (a[nr_of_iter + i].a != a[i].a) {
printf("Invalid data .a at %d %d <-> %d\n", nr_of_iter + i, a[nr_of_iter + i].a, a[i].a );
//return -1;
}
if (a[nr_of_iter + i].b != a[i].b) {
printf("Invalid data .b at %d %X <-> %X\n", nr_of_iter + i, a[nr_of_iter + i].b, a[i].b );
//return -1;
}
if (a[nr_of_iter + i].c != a[i].c) {
printf("Invalid data .c at %d %f <-> %f\n", nr_of_iter + i, a[nr_of_iter + i].c, a[i].c );
//return -1;
}
}
return 0;
}
The test passed.
I have been using your circular-byte-buffer.c for an application and it works well. However, after writing some unit tests, I discovered an unexpected behavior.
Given,
typedef struct {
uint8_t * const buffer;
int head;
int tail;
const int maxlen;
} circ_bbuf_t;
and
int circ_bbuf_push(circ_bbuf_t *c, uint8_t data)
{
int next;
next = c->head + 1; // next is where head will point to after this write.
if (next >= c->maxlen)
next = 0;
// if the head + 1 == tail, circular buffer is full. Notice that one slot
// is always left empty to differentiate empty vs full condition
if (next == c->tail)
return -1;
c->buffer[c->head] = data; // Load data and then move
c->head = next; // head to next data offset.
return 0; // return success to indicate successful push.
}
suppose we do
CIRC_BBUF_DEF(cb, 1)
You will have, cb.maxlen == 1, cb.head == 0, and cb.tail == 0
Now consider the code in circ_bbuf_push
int next;
next = c->head + 1; // next is where head will point to after this write.
if (next >= c->maxlen)
next = 0;
// if the head + 1 == tail, circular buffer is full. Notice that one slot
// is always left empty to differentiate empty vs full condition
if (next == c->tail)
return -1;
next will be incremented to 1, on the following push, next will be incremented to 2 then will be rolled over to 0, which will then fail to push. maxlen is effectively 1.
I contend that maxlen should be defined as y, and comparisons to maxlen should be > instead of >=.
Hello,
thanks for sharing, i was wondering if you can help, i would like to use the FILO.c and make it Circular buffer as well so when it's full the oldest value is overridden, similar to Circbuf.c, can you please help with that?
void hash_map_free(hash_map_t *map, hash_map_callback_t callback)
{
size_t pos = 0;
hash_map_item_t *item, *next;
while (pos < map->capacity) {
item = map->pool[pos];
while (item != NULL) {
//callback(map->pool[pos]->key, map->pool[pos]->val);
callback(item->key, item->val);
next = item->next;
safe_free(item->key);
safe_free(item);
item = next;
}
pos += 1;
}
safe_free(map->pool);
map->pool = NULL;
map->capacity = 0;
map->capacity = 0;
}
The project was compiled with gcc 4.84.
gcc prog.c -Wall -Wextra -std=c99 -pedantic
Multiple warnings occurred:
warning: ISO C forbids nested functions [-Wpedantic]
inline int x##_push_refd(t * pt) \
^
warning: ISO C forbids nested functions [-Wpedantic]
inline int x##_pop_refd(t * pt) \
warning: ISO C forbids nested functions [-Wpedantic]
inline int x##_peek_refd(t * pt) \
^
warning: ISO C forbids nested functions [-Wpedantic]
inline int x##_push_refd(t * pt) \
^
Can we avoid nesting?
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
Django
The Web framework for perfectionists with deadlines.
Laravel
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
Microsoft
Open source projects and samples from Microsoft.
Google
Google ❤️ Open Source for everyone.
Alibaba
Alibaba Open Source for everyone
D3
Data-Driven Documents codes.
Tencent
China tencent open source team.