多线程操作文件

多线程下载

多线程下载首先想到的就是先由操作系统创建一个普通文件，然后申请磁盘空间，接着由多线程将待下载的数据插入到已存在的磁盘空间中，这样是不可行的，原因就在于文件的当前操作位置是唯一的值，无法多线程操作该值。因此无法进行多线程同时对一个文件进行写入的实现；
因此就需要将这个文件在下载的时候拆为多个区间并以文件的形式暂存这些数据，然后同时下载，下载完毕后将各个区间对应的文件进行合并，最终处理为单个目标文件。
合并的话就可以直接打开要合并的两个文件，直接read、write。
相比较于单线程下载，他会更快。

示例代码

#include "pthread.h"
#include "stdio.h"
#include "unistd.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

pthread_mutex_t mutex;
int frame = 1024 * 16;

struct ThreadFuncArg
{
    int fileFD;
    char* copyFileName;
    int currentPos;             
    long fileSize;
};

void singleSplit(void* arg)
{
    ThreadFuncArg *funcArg = (ThreadFuncArg *)arg;
    int fileFD = funcArg->fileFD;
    char* copyFileName = funcArg->copyFileName;
    long fileSize = funcArg->fileSize;

    printf("fileFD -- %d, copyFileName -- %s, fileSize -- %ld\n", fileFD, copyFileName, fileSize);

    long currentPos;
    char buf[frame];
    while(1)
    {        
        currentPos = funcArg->currentPos;           // 当前需要处理的段
        funcArg->currentPos += frame;

        if(currentPos >= fileSize)       return;
        printf("currentPos -- %d, fileSize -- %ld\n", currentPos, fileSize);

        int readNum = read(fileFD, buf, frame < fileSize - currentPos ? frame : fileSize - currentPos);

        char copyFilePath[128];
        sprintf(copyFilePath, "%s_%d", copyFileName, currentPos);
        printf("copyFilePath -- %s\n", copyFilePath);
        int copyFd = open(copyFilePath, O_CREAT | O_WRONLY, 0666);
        write(copyFd, buf, readNum);
        close(copyFd);
    }

    return;
}

void* threadFunc1(void* arg)
{
    ThreadFuncArg *funcArg = (ThreadFuncArg *)arg;
    int fileFD = funcArg->fileFD;
    char* copyFileName = funcArg->copyFileName;
    long fileSize = funcArg->fileSize;

    long currentPos;
    char buf[frame];
    while(1)
    {
        // 获取区间
        pthread_mutex_lock(&mutex);
        currentPos = funcArg->currentPos;           // 当前需要处理的段
        funcArg->currentPos += frame;
        pthread_mutex_unlock(&mutex);
        if(currentPos >= fileSize)       pthread_exit(NULL);

        int readNum = read(fileFD, buf, frame < fileSize - currentPos ? frame : fileSize - currentPos);

        char copyFilePath[128];
        sprintf(copyFilePath, "%s_%d", copyFileName, currentPos);
        int copyFd = open(copyFilePath, O_CREAT | O_WRONLY, 0666);
        write(copyFd, buf, readNum);
        close(copyFd);
    }

    pthread_exit(NULL);
}

void* threadFunc2(void* arg)
{
    ThreadFuncArg *funcArg = (ThreadFuncArg *)arg;
    int fileFD = funcArg->fileFD;
    char* copyFileName = funcArg->copyFileName;
    long fileSize = funcArg->fileSize;

    long currentPos;
    char buf[frame];
    while(1)
    {
        // 获取区间
        pthread_mutex_lock(&mutex);
        currentPos = funcArg->currentPos;           // 当前需要处理的段
        funcArg->currentPos += frame;
        pthread_mutex_unlock(&mutex);
        if(currentPos >= fileSize)       pthread_exit(NULL);

        int readNum = read(fileFD, buf, frame < fileSize - currentPos ? frame : fileSize - currentPos);

        char copyFilePath[128];
        sprintf(copyFilePath, "%s_%d", copyFileName, currentPos);
        int copyFd = open(copyFilePath, O_CREAT | O_WRONLY, 0666);
        write(copyFd, buf, readNum);
        close(copyFd);
    }

    pthread_exit(NULL);
}


void combineFile(char* copyFileName)
{
    int copyFd = open(copyFileName, O_CREAT | O_WRONLY, 0666);
    int currentPos = 0;
    char buf[frame];
    while(1)
    {
        char copyFilePath_child[128];
        sprintf(copyFilePath_child, "%s_%d", copyFileName, currentPos);
        printf("combineFileName -- %s\n", copyFilePath_child);
        int copyFd_child = open(copyFilePath_child, O_RDONLY, 0666);
        if(copyFd_child == -1)            break;

        int readNum = read(copyFd_child, buf, frame);
        write(copyFd, buf, readNum);

        close(copyFd_child);
        unlink(copyFilePath_child);

        currentPos += frame;
    }
    close(copyFd);
    return;
}

int main()
{
    char* file = "/home/i/images/p4.jpg";
    char* copyFile = "./p4.jpg";

    int fileFd = open(file, O_RDONLY);
    if(fileFd == -1)
    {
        printf("open %s failed\n", file);
        return 1;
    }

    struct stat stat_buf;
    if (stat(file, &stat_buf) != 0)
	{   
        close(fileFd);
		return 1;
	}
    // 验证是否为普通文件
    if (!S_ISREG(stat_buf.st_mode))
	{
        close(fileFd);
		return 1;
    }

    // 获取文件大小
    long fileSize = stat_buf.st_size;
    printf("fileSize -- %ld\n", fileSize);

    mutex = PTHREAD_MUTEX_INITIALIZER;
    int currentCopyPos = 0;     // 当前复制位置

    // 创建两个线程去进行不同区间的复制
    pthread_t thread[2];
    ThreadFuncArg arg;
    arg.copyFileName = copyFile;
    arg.currentPos = 0;
    arg.fileFD = fileFd;
    arg.fileSize = fileSize;

#if 1
    pthread_create(&thread[0], NULL, threadFunc1, &arg);
    pthread_create(&thread[1], NULL, threadFunc2, &arg);

    // 等待两个进程
    for(int i = 0; i < 2; i++)
    {
        pthread_join(thread[i], NULL);
    }
#else
    singleSplit(&arg);
#endif
    close(fileFd);

    combineFile(copyFile);

    printf("thread finished\n");

    return 0;
}