0
我的項目需要使用mksqashfs將某些東西包裝到圖像中，當內容發生變化時，我需要用新的圖像文件替換。爲什麼每次即使內容沒有改變，mksquashfs也會導出文件更改？
在使用sqashfs之前，我使用了tarball + md5sum。內容更改，md5更改。 但是，mksquashfs每次導出圖像文件更改，甚至內容都是一樣的。
我正在步行穿過mksquashfs的所有參數，現在嘗試找到沒有運氣的問題。
A
回答
0
我也有同樣的問題，有的檢索算法後，我發現這一點：
#32 Request to add a deterministic behavior to mksquashfs
Here is a patch.
The patch adds "-deterministic" option.
Here is what the option does :
- it forces to one processor
- it forces the date globaly. The forced date is printed out and must be reused with "-forcedate" option in order to reproduced the same squashfs image.
- it sorts entries of directory as they are read.
- it delays the start of the fragment deflator thread. When this late start is active, the queue to fragment deflator thread is using an implementation of the queue with unlimited size.
Entries of directory must be sorted because the inode numbers are attributed as the files are discovered ("lookup_inode()" in "dirscan1()").
The format of the date after "-forcedate" is the number of seconds since the Epoch : the output of "date '+%s'".
With this patch, I can copy a tree of files, I can use tar or I can repack a mounted squashfs, and always have the same suqashfs file.
--- a/squashfs-tools/mksquashfs.c
+++ b/squashfs-tools/mksquashfs.c
@@ -113,6 +113,9 @@
#include "compressor.h"
#include "xattr.h"
+int forcedate = -1;
+int sort_dir_entry = 0;
+
int delete = FALSE;
int fd;
int cur_uncompressed = 0, estimated_uncompressed = 0;
@@ -342,6 +345,18 @@ struct queue {
void **data;
};
+/* struct describing queues used to pass data between threads */
+struct unlimited_queue {
+ struct unlimited_queue_node *first;
+ struct unlimited_queue_node *last;
+ pthread_mutex_t mutex;
+ pthread_cond_t any_event;
+};
+
+struct unlimited_queue_node {
+ void *data;
+ struct unlimited_queue_node* next;
+};
/* in memory uid tables */
#define ID_ENTRIES 256
@@ -362,6 +377,8 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
struct cache *reader_buffer, *writer_buffer, *fragment_buffer;
struct queue *to_reader, *from_reader, *to_writer, *from_writer, *from_deflate,
*to_frag;
+struct unlimited_queue *to_frag_unlimited = NULL;
+
pthread_t *thread, *deflator_thread, *frag_deflator_thread, progress_thread;
pthread_mutex_t fragment_mutex;
pthread_cond_t fragment_waiting;
@@ -488,6 +505,93 @@ void *queue_get(struct queue *queue)
return data;
}
+void queue_waitempty(struct queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->readp != queue->writep)
+ pthread_cond_wait(&queue->full, &queue->mutex);
+
+ pthread_cond_signal(&queue->full);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+struct unlimited_queue *unlimited_queue_init()
+{
+ struct unlimited_queue *queue = malloc(sizeof(struct unlimited_queue));
+
+ if(queue == NULL)
+ goto failed;
+
+ queue->first = NULL;
+ queue->last = NULL;
+ pthread_mutex_init(&queue->mutex, NULL);
+ pthread_cond_init(&queue->any_event, NULL);
+
+ return queue;
+
+failed:
+ BAD_ERROR("Out of memory in queue_init\n");
+}
+
+void unlimited_queue_put(struct unlimited_queue *queue, void *data)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ struct unlimited_queue_node *newNode = malloc(sizeof(struct unlimited_queue_node));
+ if(newNode == NULL){
+ BAD_ERROR("Out of memory in queue_init\n");
+ }
+
+ newNode->data = data;
+ newNode->next = NULL;
+
+ pthread_mutex_unlock(&queue->mutex);
+
+ if(queue->first == NULL){
+ queue->first = newNode;
+ } else {
+ queue->last->next = newNode;
+ }
+ queue->last = newNode;
+ pthread_cond_broadcast(&queue->any_event);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+void *unlimited_queue_get(struct unlimited_queue *queue)
+{
+ void *data;
+
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->first == NULL)
+ pthread_cond_wait(&queue->any_event, &queue->mutex);
+
+ data = queue->first->data;
+ struct unlimited_queue_node* newFirst = queue->first->next;
+ free(queue->first);
+ queue->first = newFirst;
+/* eventually
+ if(newFirst == NULL){
+ queue->last == NULL;
+ }
+*/
+
+ pthread_cond_broadcast(&queue->any_event);
+ pthread_mutex_unlock(&queue->mutex);
+
+ return data;
+}
+
+void unlimited_queue_waitempty(struct unlimited_queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->first != NULL)
+ pthread_cond_wait(&queue->any_event, &queue->mutex);
+
+ pthread_mutex_unlock(&queue->mutex);
+}
/* Cache status struct. Caches are used to keep
track of memory buffers passed between different threads */
@@ -1852,7 +1956,11 @@ void write_fragment()
fragment_data->block = fragments;
fragment_table[fragments].unused = 0;
fragments_outstanding ++;
- queue_put(to_frag, fragment_data);
+ if(to_frag_unlimited){
+ unlimited_queue_put(to_frag_unlimited, fragment_data);
+ }else{
+ queue_put(to_frag, fragment_data);
+ }
fragments ++;
fragment_size = 0;
pthread_mutex_unlock(&fragment_mutex);
@@ -2613,8 +2721,15 @@ void *frag_deflator(void *arg)
BAD_ERROR("frag_deflator:: compressor_init failed\n");
while(1) {
+ struct file_buffer *file_buffer = NULL;
+
+ if(to_frag_unlimited){
+ file_buffer = unlimited_queue_get(to_frag_unlimited);
+ }else{
+ file_buffer = queue_get(to_frag);
+ }
+
int c_byte, compressed_size;
- struct file_buffer *file_buffer = queue_get(to_frag);
struct file_buffer *write_buffer =
cache_get(writer_buffer, file_buffer->block +
FRAG_INDEX, 1);
@@ -3594,7 +3709,11 @@ void dir_scan(squashfs_inode *inode, char *pathname,
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
buf.st_uid = getuid();
buf.st_gid = getgid();
- buf.st_mtime = time(NULL);
+ if(forcedate != -1){
+ buf.st_mtime = forcedate;
+ }else{
+ buf.st_mtime = time(NULL);
+ }
buf.st_dev = 0;
buf.st_ino = 0;
dir_ent->inode = lookup_inode(&buf);
@@ -3634,11 +3753,23 @@ void dir_scan(squashfs_inode *inode, char *pathname,
dir_ent->inode->type = SQUASHFS_DIR_TYPE;
}
+struct file_dir_name{
+ char* filename;
+ char* dirname;
+};
+
+int compare_file_dir_name(const void* a, const void* b){
+ return strcmp((*(struct file_dir_name**)a)->filename, (*(struct file_dir_name**)b)->filename);
+}
+
+#define BASE_ALLOC_INCREMENT 128
struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
int (_readdir)(char *, char *, struct dir_info *))
{
- char filename[8192], dir_name[8192];
+ char filename2[8192], dir_name2[8192];
+ char *filename = filename2;
+ char *dir_name = dir_name2;
struct dir_info *dir = scan1_opendir(pathname);
if(dir == NULL) {
@@ -3646,18 +3777,69 @@ struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
goto error;
}
- while(_readdir(filename, dir_name, dir) != FALSE) {
+ int base_count = 0;
+ void** base = NULL;
+
+ if(sort_dir_entry){
+ int base_allocated = BASE_ALLOC_INCREMENT;
+ base = malloc(base_allocated * sizeof(void*));
+
+ while(_readdir(filename2, dir_name2, dir) != FALSE) {
+ struct file_dir_name* new_element = malloc(sizeof(struct file_dir_name));
+ if(new_element == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+ new_element->filename = strdup(filename2);
+ if(new_element->filename == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+ new_element->dirname = strdup(dir_name2);
+ if(new_element->dirname == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+
+ if(base_count >= base_allocated){
+ base_allocated += BASE_ALLOC_INCREMENT;
+ base = realloc(base, base_allocated * sizeof(void*));
+ if(base == NULL){
+ BAD_ERROR("No more Memory !\n");
+ }
+ }
+
+ base[base_count] = new_element;
+ ++base_count;
+ }
+
+ qsort(base, base_count, sizeof(void*), compare_file_dir_name);
+ }
+
+ int i = 0;
+
+ while(1){
+ if(sort_dir_entry){
+ if(i >= base_count){
+ break;
+ }
+ struct file_dir_name* cur_element = (struct file_dir_name*)base[i];
+ dir_name = cur_element->dirname;
+ filename = cur_element->filename;
+ }else{
+ if(_readdir(filename, dir_name, dir) == FALSE) {
+ break;
+ }
+ }
+
struct dir_info *sub_dir;
struct stat buf;
struct pathnames *new;
if(strcmp(dir_name, ".") == 0 || strcmp(dir_name, "..") == 0)
- continue;
+ goto loop_continue;
if(lstat(filename, &buf) == -1) {
ERROR("Cannot stat dir/file %s because %s, ignoring",
filename, strerror(errno));
- continue;
+ goto loop_continue;
}
if((buf.st_mode & S_IFMT) != S_IFREG &&
@@ -3669,29 +3851,40 @@ struct dir_info *dir_scan1(char *pathname, struct pathnames *paths,
(buf.st_mode & S_IFMT) != S_IFSOCK) {
ERROR("File %s has unrecognised filetype %d, ignoring"
"\n", filename, buf.st_mode & S_IFMT);
- continue;
+ goto loop_continue;
}
if(old_exclude) {
if(old_excluded(filename, &buf))
- continue;
+ goto loop_continue;
} else {
if(excluded(paths, dir_name, &new))
- continue;
+ goto loop_continue;
}
if((buf.st_mode & S_IFMT) == S_IFDIR) {
sub_dir = dir_scan1(filename, new, scan1_readdir);
if(sub_dir == NULL)
- continue;
+ goto loop_continue;
dir->directory_count ++;
} else
sub_dir = NULL;
add_dir_entry(dir_name, filename, sub_dir, lookup_inode(&buf),
dir);
+ loop_continue:
+ if(sort_dir_entry){
+ free(dir_name);
+ free(filename);
+ free(base[i]);
+ base[i] = NULL;
+ i++;
+ }
}
+ if(sort_dir_entry){
+ free(base);
+ }
scan1_freedir(dir);
error:
@@ -3777,7 +3970,11 @@ struct dir_info *dir_scan2(struct dir_info *dir, struct pseudo *pseudo)
buf.st_gid = pseudo_ent->dev->gid;
buf.st_rdev = makedev(pseudo_ent->dev->major,
pseudo_ent->dev->minor);
- buf.st_mtime = time(NULL);
+ if(forcedate != -1){
+ buf.st_mtime = forcedate;
+ }else{
+ buf.st_mtime = time(NULL);
+ }
buf.st_ino = pseudo_ino ++;
if(pseudo_ent->dev->type == 'f') {
@@ -4039,6 +4236,7 @@ void add_old_root_entry(char *name, squashfs_inode inode, int inode_number,
old_root_entry[old_root_entries++].inode.root_entry = TRUE;
}
+int late_fragment_thread_start = 0;
void initialise_threads(int readb_mbytes, int writeb_mbytes,
int fragmentb_mbytes)
@@ -4046,7 +4244,7 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
int i;
sigset_t sigmask, old_mask;
int reader_buffer_size = readb_mbytes << (20 - block_log);
- int fragment_buffer_size = fragmentb_mbytes << (20 - block_log);
+ int fragment_buffer_size = (fragmentb_mbytes == -1 ? -1 : fragmentb_mbytes << (20 - block_log));
/*
* writer_buffer_size is global because it is needed in
@@ -4095,10 +4293,16 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
to_writer = queue_init(writer_buffer_size);
from_writer = queue_init(1);
from_deflate = queue_init(reader_buffer_size);
- to_frag = queue_init(fragment_buffer_size);
+
+ if(fragment_buffer_size == -1){
+ to_frag_unlimited = unlimited_queue_init();
+ }else{
+ to_frag = queue_init(fragment_buffer_size);
+ }
+
reader_buffer = cache_init(block_size, reader_buffer_size);
writer_buffer = cache_init(block_size, writer_buffer_size);
- fragment_buffer = cache_init(block_size, fragment_buffer_size);
+ fragment_buffer = cache_init(block_size, fragment_buffer_size != -1 ? fragment_buffer_size : FRAGMENT_BUFFER_DEFAULT << (20 - block_log));
pthread_create(&thread[0], NULL, reader, NULL);
pthread_create(&thread[1], NULL, writer, NULL);
pthread_create(&progress_thread, NULL, progress_thrd, NULL);
@@ -4109,9 +4313,11 @@ void initialise_threads(int readb_mbytes, int writeb_mbytes,
if(pthread_create(&deflator_thread[i], NULL, deflator, NULL) !=
0)
BAD_ERROR("Failed to create thread\n");
- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
- NULL) != 0)
- BAD_ERROR("Failed to create thread\n");
+ if(!late_fragment_thread_start){
+ if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
+ NULL) != 0)
+ BAD_ERROR("Failed to create thread\n");
+ }
}
printf("Parallel mksquashfs: Using %d processor%s\n", processors,
@@ -4701,6 +4907,31 @@ int main(int argc, char *argv[])
"megabyte or larger\n", argv[0]);
exit(1);
}
+ } else if(strcmp(argv[i], "-forcedate") == 0) {
+ if(++i == argc) {
+ ERROR("%s: -forcedate missing fake date\n", argv[0]);
+ exit(1);
+ }
+ forcedate = strtol(argv[i], &b, 10);
+ if(*b != '\0') {
+ ERROR("%s: -forcedate invalid parameter\n", argv[0]);
+ exit(1);
+ }
+ } else if(strcmp(argv[i], "-deterministic") == 0) {
+ if(forcedate == -1){
+ forcedate = time(NULL);
+ }
+ processors = 1;
+ late_fragment_thread_start = 1;
+ fragmentb_mbytes = -1;
+ if(sort_dir_entry == 0){
+ sort_dir_entry = 1;
+ }
+ } else if(strcmp(argv[i], "-late-fragment-thread-start") == 0) {
+ late_fragment_thread_start = 1;
+ fragmentb_mbytes = -1;
+ } else if(strcmp(argv[i], "-unlimited-fragment-queue") == 0) {
+ fragmentb_mbytes = -1;
} else if(strcmp(argv[i], "-b") == 0) {
if(++i == argc) {
ERROR("%s: -b missing block size\n", argv[0]);
@@ -5292,7 +5523,11 @@ printOptions:
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
always_use_fragments, duplicate_checking, exportable,
no_xattrs, comp_opts);
- sBlk.mkfs_time = time(NULL);
+ if(forcedate != -1){
+ sBlk.mkfs_time = forcedate;
+ } else {
+ sBlk.mkfs_time = time(NULL);
+ }
restore_filesystem:
if(progress && estimated_uncompressed) {
@@ -5301,6 +5536,14 @@ restore_filesystem:
}
write_fragment();
+
+ if(late_fragment_thread_start){
+ queue_waitempty(from_reader);
+ if(pthread_create(&frag_deflator_thread[0], NULL, frag_deflator,
+ NULL) != 0)
+ BAD_ERROR("Failed to create thread\n");
+ }
+
sBlk.fragments = fragments;
if(!restoring) {
unlock_fragments();
@@ -5340,7 +5583,8 @@ restore_filesystem:
write_destination(fd, SQUASHFS_START, sizeof(sBlk), &sBlk);
if(!nopad && (i = bytes & (4096 - 1))) {
- char temp[4096] = {0};
+ char temp[4096];
+ memset(&temp, 0, 4096);
write_destination(fd, bytes, 4096 - i, temp);
}
@@ -5401,6 +5645,9 @@ restore_filesystem:
printf("Number of directories %d\n", dir_count);
printf("Number of ids (unique uids + gids) %d\n", id_count);
printf("Number of uids %d\n", uid_count);
+ if(forcedate != -1){
+ printf("forcedate was %d\n", forcedate);
+ }
for(i = 0; i < id_count; i++) {
if(id_table[i]->flags & ISA_UID) {
0
有在github另一修改mksquashfs：
https://github.com/dannyp11/squashfs4.3.git
這叉增加 - 沒有日期的選項，使mksquashfs創建壓縮文件沒有使日期信息。因此，如果內容保持不變，結果壓扁文件具有相同的chksum。
diff -ruNa squashfs4.3/README squashfs4.3-new/README
--- squashfs4.3/README 2014-05-12 13:17:59.000000000 -0700
+++ squashfs4.3-new/README 2017-08-01 23:05:44.428051586 -0700
@@ -127,6 +127,9 @@
-keep-as-directory if one source directory is specified, create a root
directory containing that directory, rather than the
contents of the directory
+-no-date do not store the date in the squash file
+ (-noappend and -no-fragments is also activated)
+ this way, the squash always has the same cksum
Filesystem filter options:
-p <pseudo-definition> Add pseudo file definition
diff -ruNa squashfs4.3/squashfs-tools/mksquashfs.c squashfs4.3-new/squashfs-tools/mksquashfs.c
--- squashfs4.3/squashfs-tools/mksquashfs.c 2014-05-12 15:18:20.000000000 -0700
+++ squashfs4.3-new/squashfs-tools/mksquashfs.c 2017-08-01 21:26:03.092224142 -0700
@@ -98,6 +98,7 @@
int use_regex = FALSE;
int nopad = FALSE;
int exit_on_error = FALSE;
+int no_date = FALSE;
long long global_uid = -1, global_gid = -1;
@@ -3139,7 +3140,7 @@
buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
buf.st_uid = getuid();
buf.st_gid = getgid();
- buf.st_mtime = time(NULL);
+ buf.st_mtime = (!no_date)? time(NULL) : 0;
buf.st_dev = 0;
buf.st_ino = 0;
dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0);
@@ -3526,7 +3527,7 @@
buf.st_gid = pseudo_ent->dev->gid;
buf.st_rdev = makedev(pseudo_ent->dev->major,
pseudo_ent->dev->minor);
- buf.st_mtime = time(NULL);
+ buf.st_mtime = (!no_date)? time(NULL) : 0;
buf.st_ino = pseudo_ino ++;
if(pseudo_ent->dev->type == 'd') {
@@ -5262,6 +5263,12 @@
else if(strcmp(argv[i], "-keep-as-directory") == 0)
keep_as_directory = TRUE;
+ else if (strcmp(argv[i], "-no-date") == 0) {
+ no_date = TRUE;
+ delete = TRUE;
+ no_fragments = TRUE;
+ }
+
else if(strcmp(argv[i], "-exit-on-error") == 0)
exit_on_error = TRUE;
@@ -5315,6 +5322,9 @@
ERROR("\t\t\tdirectory containing that directory, "
"rather than the\n");
ERROR("\t\t\tcontents of the directory\n");
+ ERROR("-no-date\t\tdo not store the date in the squash file\n");
+ ERROR("\t\t\t(-noappend and -no-fragments is also activated)\n");
+ ERROR("\t\t\tthis way, the squash always has the same cksum\n");
ERROR("\nFilesystem filter options:\n");
ERROR("-p <pseudo-definition>\tAdd pseudo file "
"definition\n");
@@ -5717,7 +5727,7 @@
sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
always_use_fragments, duplicate_checking, exportable,
no_xattrs, comp_opts);
- sBlk.mkfs_time = time(NULL);
+ sBlk.mkfs_time = (!no_date) ? time(NULL) : 0;
disable_info();
相關問題
- 1. 德爾福XE2改變DFM即使什麼也沒有改變
- 2. Visual Studio每次編譯一些文件（即使沒有更改）
- 3. 爲什麼每次重建都會更改二進制文件？
- 4. SVN每次提交或合併都會添加文件，即使沒有更改
- 5. QSettings是否再次寫入設置文件，即使內容沒有改變？
- 6. 爲什麼每次編譯時內存地址都會改變？
- 7. 爲什麼更改innerText值也會改變innerHTML？
- 8. 即使沒有值更改，mysql current_timestamp列也會更新
- 9. 即使改變site.css，爲什麼樣式不會改變？
- 10. 即使沒有內容輸入，爲什麼網頁表單也會提交？
- 11. 即使在php.ini文件中更改圖像也沒有上傳
- 12. 即使在更改內容後，Chrome擴展程序彈出框也會重置
- 13. 爲什麼這個批處理變量即使設置也不會改變？
- 14. 爲什麼更改實例變量也會改變靜態變量？
- 15. 爲什麼有些細胞即使不符合條件也會被修改？
- 16. 我更改代碼隱藏文件，什麼也沒有發生
- 17. 爲什麼Rails會保存會話，即使沒有任何更改？
- 18. 將STDERR重定向到Bash文件中會導致即使沒有錯誤也會創建文件。爲什麼？
- 19. 爲什麼即使在const函數中值也被改變？
- 20. Appengine爲什麼有時會更改內容類型？
- 21. 改變ArrayAdapter的內容沒有更新
- 22. 爲什麼切片的內容在GO中沒有改變？
- 23. 爲什麼git會重新提交所有文件，即使是我沒有更改的文件
- 24. 即使條件不滿足，爲什麼狀態會改變？
- 25. 即使我改變它，我的主窗口也不會改變？
- 26. 爲什麼我的DB項目的.dbmdl文件更改，即使我沒有改變項目？
- 27. 即使更改my.cnf文件後，group_concat_max_length系統變量也未更改
- 28. 即使值未改變，jQuery UI滑塊「更改」事件也會觸發
- 29. 即使沒有SVN更改，Jenkins爲什麼觸發IVY？
- 30. 爲什麼每個版本都會改變exe文件？