项目场景：
项目需求，需要做一个基于边缘端的人脸识别远程监控摄像头并在网页前端展示 ，这里采用国产香橙派作为边缘计算终端，安装ubuntu系统，系统中采用v4l2接口对摄像头进行获取，当客户端通过网页进行请求时，服务器通过http服务的形式将一帧帧图像发送给客户端，只要一秒钟能够传送25帧左右，展示效果就是在网页端播放视频：
问题描述1
怎样从摄像头里面获取帧数据，这里我们将USB摄像头连接在开发板上：
可以看到，确实是有video0这个摄像头，该摄像头就是外接的USB摄像头
解决方案1：
采用V4L2接口通过中断将内核状态读取到的数据映射到用户空间：
以下代码是将内核空间与用户空间进行映射
for(int i = 0; i <4;i++) {
mapbuffer.index = i;
ret = ioctl(fd,VIDIOC_QUERYBUF,&mapbuffer); //从内核空间中查询一个空间作映射
if (ret < 0)
{
perror("查询内核空间失败");
}
//映射到用户空间
mptr[i] = (unsigned char *)mmap(NULL,mapbuffer.length,PROT_READ|PROT_WRITE,MAP_SHARED,fd,mapbuffer.m.offset);
size[i] = mapbuffer.length; //保存映射长度用于后期释放
//查询后通知内核已经放回
ret = ioctl(fd,VIDIOC_QBUF,&mapbuffer);
if (ret < 0)
{
perror("放回失败");
}
}
以下代码是通过中断获取内核态的数据，并在用户态对数据进行内存拷贝，即将数据拷贝到数组，方便其他线程进行数据处理和数据展示
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //每个结构体都需要设置type为这个参赛要记住
ret = ioctl(fd,VIDIOC_DQBUF,&readbuffer);
int read_len = mapbuffer.length;
int send_len = 1024;
//printf("%d \n",read_len);
if (ret < 0)
{
perror("读取数据失败");
}
if(1)
{
void *ptr = mptr[readbuffer.index];
memcpy(jpg_buff, ptr, readbuffer.length);
}问题描述2
在网页前端播放视频，首先视频是一帧帧图像组成的，这里我们每间隔一个短时间就发送一帧，同时http 客户端和服务端采用keep-live长连接形式。为了加快速度，这里我采用了两个线程，现成1读取视频，并将图像帧放在一个公共数组中，线程2将公共数组中的数据通过tcp传输发送给客户端。
同时，线程1和线程之间采用的是条件变量进行同步
解决方案2：
多线程共享全局变量加快视频读取速度和图像发送速度：
线程1
pthread_mutex_lock(&lock);
void *ptr = mptr[readbuffer.index];
memcpy(jpg_buff, ptr, readbuffer.length);//将读取到的图像拷贝到字符数组
pthread_mutex_unlock(&lock);
pthread_cond_signal(&hasNode);
线程2
pthread_cond_wait(&hasNode,&lock);
void *ppptr = jpg_buff;
pthread_mutex_unlock(&lock);这里的jpg_buff是共享全局数组
问题描述3
网页长连接设置问题：
服务器端在返回报文头时，加上
"Content-Type:multipart/x-mixed-replace;
即表示当前连接是基于长连接
解决方案：
提示：这里填写该问题的具体解决方案：
下面是报文头的内容
/*
HTTP长连接处理客户端请求
"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n"
*/这是程序运行时的效果图：
需要在与开发板同一个局域网的终端上网址中键入：192.168.0.105:8080
其中192.168.0.105是开发板的ip地址
最后附上总代码
#include <stdio.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <assert.h>
#include <fcntl.h>
#include <jpeglib.h>
#include <linux/fb.h>
#include <linux/videodev2.h>
#include <netinet/in.h>
#include <netinet/ip.h> /* superset of previous */
#include <pthread.h>
#include "sd_usb_pic.h"
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/types.h> /* See NOTES */
#include <unistd.h>
#include <sys/time.h>
#define HTTP_PORT 8080 //HTTP服务器端口号
pthread_mutex_t mutex;
pthread_cond_t hasNode = PTHREAD_COND_INITIALIZER;
pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
char jpg_buff[614400]; //保存从USB摄像头读取的图像矩阵
int send_html_cnt = 0 ;
/*
服务端响应客户端请求
"HTTP/1.1 200 OK\r\n"
"Content-type:image/jpeg\r\n"
"Content-Length:1234\r\n"
"\r\n"
形参：c_fd --客户端套接字
type --文件类型
file --要发送的文件
返回值：0成功，其它失败
*/
void *ppp_tr = NULL;
int fd_fb;
static struct fb_var_screeninfo var; /* LCD可变参数 */
static unsigned int *fb_base = NULL; /* Framebuffer映射基地址 */
int lcd_w = 800 ,lcd_h= 480; //定义显示器分辨率
/*
形参：c_fd --客户端套接字
type --文件类型
file --要发送的文件
返回值：0成功，其它失败
*/
int Http_SendData(int c_fd,const char *type,const char *file)
{
int fd=open(file,O_RDONLY);//打开文件
if(fd<0)return -1;//打开文件失败
struct stat statbuf;
fstat(fd,&statbuf);
if(statbuf.st_size<=0)
{
close(fd);
return -2;
}
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Content-type:%s\r\n"
"Content-Length:%ld\r\n"
"\r\n",type,statbuf.st_size);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
close(fd);
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
while(1)
{
size=read(fd,buff,sizeof(buff));
if(write(c_fd,buff,size)!=size)break;//发送失败
if(size!=sizeof(buff))break;//发送完成
}
close(fd);
return 0;
}
int Http_SendPic(int c_fd,const char *type,const char *file)
{
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Content-type:%s\r\n"
"Content-Length:%ld\r\n"
"\r\n",type,614400);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
int cnt = 0 ;
pthread_mutex_lock(&mutex);
void *ppptr = jpg_buff;
while(1)
{
int size= 1024;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
pthread_mutex_unlock(&mutex);
return 0;
}
int Http_SendPic1(int c_fd,const char *type,const char *file)
{
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Server:LiMeng \r\n"
"Content-type:image/jpeg\r\n"
"Content-Length:%ld\r\n"
"\r\n",614400);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
int cnt = 0 ;
pthread_mutex_lock(&mutex);
void *ppptr = jpg_buff;
while(1)
{
int size= 1024;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
pthread_mutex_unlock(&mutex);
return 0;
}
/*
HTTP长连接处理客户端请求
"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n"
*/
int Http_Content(int c_fd)
{
char buff[1024]={0};
/*建立长连接*/
snprintf(buff,sizeof(buff),"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n");
if(write(c_fd,buff,strlen(buff))!=strlen(buff))return -1;//发送报文头失败
int jpe_image_size = 614400;//保存jpeg图像大小
//int send_html_cnt = 0 ;
clock_t t1;
t1 = clock();
char save_name[10];
int save_idx = 0 ;
struct timeval start_time, end_time;
while(1)
{
//auto beg = clock();
t1 = clock();
pthread_cond_wait(&hasNode,&lock);
//printf("wait time is %d us \n", (clock() - t1) );
auto beg = clock();
gettimeofday(&start_time, NULL);
/*
(1)响应报文头
"Content-type:image/jpeg\r\n"
"Content-Length:666\r\n"
"\r\n"
*/
snprintf(buff,sizeof(buff), "Content-type:image/jpeg\r\n"
"Content-Length:%d\r\n"
"\r\n",jpe_image_size);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -2;//响应报文头失败
}
/*发送jpg图像数据*/
//pthread_mutex_lock(&mutex);//互斥锁上锁
//pthread_cond_wait(&hasNode,&lock);
void *ppptr = jpg_buff;
//void *ppptr = ppp_tr;
//sprintf(save_name,"my_%d.jpg",save_idx++);
//FILE *file=fopen(save_name, "w");
//fwrite(ppptr , 614400, 1,file);
//close(file);
int cnt = 0 ;
while(1)
{
int size= 1024 * 600;
//int size= 1024 ;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
//pthread_mutex_unlock(&mutex);//互斥锁上锁
//pthread_mutex_unlock(&lock);
//sleep(20);
/*
(3)发送间隔字符串
"\r\n"
"--boundarydonotcross\r\n"
*/
strcpy(buff,"\r\n--boundarydonotcross\r\n");
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
break;//发送间隔符失败
}
auto end = clock();
gettimeofday(&end_time, NULL);
double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
//printf("sendtime is %d us \n", (end - beg) );
//printf("send cnt is %d us \n", send_html_cnt );
send_html_cnt++;
//pthread_mutex_unlock(&lock);
//usleep(40000);
//auto beg = clock();
//printf("sendtime is %d ms \n", (end - beg) );
printf("sendtime is %d ms \n", timeuse );
pthread_mutex_unlock(&lock);
usleep(1);
}
return -4;//发送图像数据失败
}
/*线程工作函数*/
void *pth_work(void *arg)
{
int c_fd=*(int *)arg;
free(arg);
char buff[1024]={0};
int size;
size=read(c_fd,buff,sizeof(buff)-1);
if(size<=0)
{
close(c_fd);
pthread_exit(NULL);
}
buff[size]='\0';
printf("buff=%s\n",buff);
if(strstr(buff,"GET / HTTP/1.1"))//请求网页文件
{
Http_SendData(c_fd,"text/html","./html/image.html");
}
else if(strstr(buff,"GET /1.bmp HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/1.bmp");
}
else if(strstr(buff,"GET /my.jpg HTTP/1.1"))
{
//Http_SendData(c_fd,"application/x-jpg","./html/my.jpg");
//Http_SendPic(c_fd,"application/x-jpg","./html/my.jpg");
Http_Content(c_fd);
}
else if(strstr(buff,"GET /my_32.jpg HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/my_32.jpg");
}
else if(strstr(buff,"GET /100.bmp HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/100.bmp");
}
else if(strstr(buff,"GET /favicon.ico HTTP/1.1"))
{
Http_SendData(c_fd,"image/x-icon","./html/wmp.ico");
}
else
{
Http_SendData(c_fd,"application/x-jpg","./html/limeng.jpg");
//Http_SendData(c_fd,"application/x-bmp","./html/my.jpg");
}
close(c_fd);
//printf("22222222222222222222222 \n");
pthread_exit(NULL);
}
int generate_pic()
{
int fd = open("/dev/video0",O_RDWR); //打开摄像头设备
if (fd < 0)
{
perror("打开设备失败");
return -1;
}
struct v4l2_format vfmt;
vfmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //摄像头采集
vfmt.fmt.pix.width = 640; //设置摄像头采集参数，不可以任意设置
vfmt.fmt.pix.height = 480;
vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG; //设置视频采集格式为mjpg格式
int ret = ioctl(fd,VIDIOC_S_FMT,&vfmt);
if (ret < 0)
{
perror("设置格式失败1");
}
//申请内核空间
struct v4l2_requestbuffers reqbuffer;
reqbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
reqbuffer.count = 4; //申请4个缓冲区
reqbuffer.memory = V4L2_MEMORY_MMAP; //映射方式
ret = ioctl(fd,VIDIOC_REQBUFS,&reqbuffer);
if (ret < 0)
{
perror("申请空间失败");
}
//映射
unsigned char *mptr[4];//保存映射后用户空间的首地址
unsigned int size[4];
struct v4l2_buffer mapbuffer;
//初始化type和index
mapbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
for(int i = 0; i <4;i++) {
mapbuffer.index = i;
ret = ioctl(fd,VIDIOC_QUERYBUF,&mapbuffer); //从内核空间中查询一个空间作映射
if (ret < 0)
{
perror("查询内核空间失败");
}
//映射到用户空间
mptr[i] = (unsigned char *)mmap(NULL,mapbuffer.length,PROT_READ|PROT_WRITE,MAP_SHARED,fd,mapbuffer.m.offset);
size[i] = mapbuffer.length; //保存映射长度用于后期释放
//查询后通知内核已经放回
ret = ioctl(fd,VIDIOC_QBUF,&mapbuffer);
if (ret < 0)
{
perror("放回失败");
}
}
//开始采集
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd,VIDIOC_STREAMON,&type);
if (ret < 0){ perror("开启失败");}
//定义一个空间存储解码后的rgb
int sum = 0;//这里是用来控制读取的帧的数量 实际场景下 可以不要这个限制
int read_cnt = 0 ;//记录读取图像线程
char save_name[10];
int save_idx = 0 ;
struct timeval start_time, end_time;
while(1)
{
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //每个结构体都需要设置type为这个参赛要记住
ret = ioctl(fd,VIDIOC_DQBUF,&readbuffer);
int read_len = mapbuffer.length;
int send_len = 1024;
//printf("%d \n",read_len);
if (ret < 0)
{
perror("读取数据失败");
}
if(1)
{
//void *ptr = mptr[readbuffer.index];
//pthread_mutex_lock(&mutex);
//gettimeofday(&start_time, NULL);
pthread_mutex_lock(&lock);
void *ptr = mptr[readbuffer.index];
///ppp_tr = mptr[readbuffer.index];
//auto beg = clock();
//printf("read frame %d \n",beg);
//printf("read frame %d %d %d \n",read_cnt++,send_html_cnt,beg);
//pthread_mutex_lock(&mutex);
memcpy(jpg_buff, ptr, readbuffer.length);//将读取到的图像拷贝到字符数组
//ppp_tr = ptr ;
//auto end = clock();
//gettimeofday(&end_time, NULL);
//double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
sprintf(save_name,"my_%d.jpg",save_idx++);
//int fd = open(save_name, O_CREAT |O_WRONLY|O_TRUNC,0666);
//char buf[1024*600]={0};
//FILE *file=fopen(save_name, "w");
//fwrite(mptr[readbuffer.index] , readbuffer.length, 1,file);
//fwrite(ptr , readbuffer.length, 1,file);
//close(file);
//write(fd,buf,1024 * 600);
//close(fd);
pthread_mutex_unlock(&lock);
pthread_cond_signal(&hasNode);
//gettimeofday(&end_time, NULL);
//pthread_mutex_unlock(&mutex);
auto end = clock();
//printf("runtime is %f \n", (end - beg) / CLOCKS_PER_SEC );
//printf("runtime is %f \n", (end - beg) );
printf("runtime is %d ms \n", timeuse );
//sleep(20);
usleep(1);
}
//通知内核使用完毕
ret = ioctl(fd, VIDIOC_QBUF, &readbuffer);
if(ret < 0)
{
perror("放回队列失败");
}
}
//停止采集
ret = ioctl(fd,VIDIOC_STREAMOFF,&type);
//释放映射
for(int i=0; i<4; i)munmap(mptr[i], size[i]);
close(fd); //关闭文件
return 0;
err1:
close(fd_fb);
return -1;
}
void shuijiao()
{
struct timeval start_time, end_time;
while(1){
gettimeofday(&start_time, NULL);
pthread_cond_wait(&hasNode,&lock);
usleep(10000);
pthread_mutex_unlock(&lock);
gettimeofday(&end_time, NULL);
double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
//printf("shuijiao time is %f ms \n", timeuse * 1000 );
}
}
int main()
{
int sockfd=socket(AF_INET,SOCK_STREAM,0);
if(sockfd==-1)
{
printf("创建网络套接字失败\n");
return 0;
}
/*允许绑定已使用的端口号*/
int on = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
/*绑定端口号*/
struct sockaddr_in s_addr=
{
.sin_family=AF_INET,//IPV4
.sin_port=htons(HTTP_PORT),
.sin_addr.s_addr=INADDR_ANY
};
if(bind(sockfd,(const struct sockaddr *)&s_addr,sizeof(s_addr)))
{
printf("绑定端口号失败\n");
return 0;
}
/*设置监听数量*/
listen(sockfd,100);
/*等待客户端连接*/
struct sockaddr_in c_addr;
socklen_t len=sizeof(c_addr);
int c_fd;
pthread_t pthid;
int *p=NULL;
pthread_t pthid1;
pthread_create(&pthid1,NULL,generate_pic,NULL);
pthread_detach(pthid1);//设置为分离属性
//shuijiao();
//while(1){shuijiao();}
while(1)
{
c_fd=accept(sockfd,(struct sockaddr *)&c_addr,&len);
if(c_fd==-1)
{
printf("客户端连接失败\n");
continue;
}
printf("套接字 : %d 连接成功,%s:%d\n",c_fd,inet_ntoa(c_addr.sin_addr),ntohs(c_addr.sin_port));
p=malloc(4);
*p=c_fd;
pthread_create(&pthid,NULL,pth_work,p);
pthread_detach(pthid);//设置为分离属性
}
}
代码的编译命令是
gcc serv.c -o serv -ljpeg
