New features!

- Packing .blend files

If you work a lot with dynamic linked .blend files ("Libraries"), it's always hard to
share your work with others (or for bug reports!).
This new option packs all used external .blend files, and - on save - combines it together
in one .blend file. You can save that file on any location.

Loading a packed .blend file then loads all library data usual - not editable.

Just use unpack to save out all linked .blend files. This will only save out the files
according the directory structure as was used on linking - relative to the current .blend.
It will create new directories, so be careful with unpacking when relative paths go up.

This feature also works fine for linked compressed .blend files.

It also works for many levels deep linked .blend hierarchies.

Access is hidden for now - I need to get some people to give it serious testing first.
You can find the options via spacebar search (try pack or unpack).

- Packed data and Undo

Now all packed data is excluded from the Undo buffer storage. Keeps undo memory smaller
and makes faster redo possible.
This commit is contained in:
Ton Roosendaal 2012-12-27 15:07:19 +00:00
parent 1f4fda10ce
commit 16411da41e
11 changed files with 315 additions and 38 deletions

View File

@ -48,6 +48,7 @@ struct PackedFile *newPackedFile(struct ReportList *reports, const char *filenam
struct PackedFile *newPackedFileMemory(void *mem, int memlen);
void packAll(struct Main *bmain, struct ReportList *reports);
void packLibraries(struct Main *bmain, struct ReportList *reports);
/* unpack */
char *unpackFile(struct ReportList *reports, const char *abs_name, const char *local_name, struct PackedFile *pf, int how);
@ -55,6 +56,7 @@ int unpackVFont(struct ReportList *reports, struct VFont *vfont, int how);
int unpackSound(struct Main *bmain, struct ReportList *reports, struct bSound *sound, int how);
int unpackImage(struct ReportList *reports, struct Image *ima, int how);
void unpackAll(struct Main *bmain, struct ReportList *reports, int how);
int unpackLibraries(struct Main *bmain, struct ReportList *reports);
int writePackedFile(struct ReportList *reports, const char *filename, struct PackedFile *pf, int guimode);

View File

@ -588,8 +588,11 @@ void BKE_bpath_traverse_id(Main *bmain, ID *id, BPathVisitor visit_cb, const int
case ID_LI:
{
Library *lib = (Library *)id;
if (rewrite_path_fixed(lib->name, visit_cb, absbase, bpath_user_data)) {
BKE_library_filepath_set(lib, lib->name);
/* keep packedfile paths always relative to the blend */
if (lib->packedfile == NULL) {
if (rewrite_path_fixed(lib->name, visit_cb, absbase, bpath_user_data)) {
BKE_library_filepath_set(lib, lib->name);
}
}
break;
}

View File

@ -43,6 +43,7 @@
#include "MEM_guardedalloc.h"
#include "DNA_image_types.h"
#include "DNA_ID.h"
#include "DNA_sound_types.h"
#include "DNA_vfont_types.h"
#include "DNA_packedFile_types.h"
@ -226,6 +227,7 @@ PackedFile *newPackedFile(ReportList *reports, const char *filename, const char
return (pf);
}
/* no libraries for now */
void packAll(Main *bmain, ReportList *reports)
{
Image *ima;
@ -538,6 +540,41 @@ int unpackImage(ReportList *reports, Image *ima, int how)
return(ret_value);
}
int unpackLibraries(Main *bmain, ReportList *reports)
{
Library *lib;
char *newname;
int ret_value = RET_ERROR;
for (lib = bmain->library.first; lib; lib = lib->id.next) {
if (lib->packedfile && lib->name[0]) {
newname = unpackFile(reports, lib->filepath, lib->filepath, lib->packedfile, PF_WRITE_ORIGINAL);
if (newname != NULL) {
ret_value = RET_OK;
printf("Saved .blend library: %s\n", newname);
freePackedFile(lib->packedfile);
lib->packedfile = NULL;
MEM_freeN(newname);
}
}
}
return(ret_value);
}
void packLibraries(Main *bmain, ReportList *reports)
{
Library *lib;
for (lib = bmain->library.first; lib; lib = lib->id.next)
if (lib->packedfile == NULL)
lib->packedfile = newPackedFile(reports, lib->name, bmain->name);
}
void unpackAll(Main *bmain, ReportList *reports, int how)
{
Image *ima;

View File

@ -311,6 +311,9 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain, const char *filename, MemFil
/* makes lookup of existing video clips in old main */
blo_make_movieclip_pointer_map(fd, oldmain);
/* makes lookup of existing video clips in old main */
blo_make_packed_pointer_map(fd, oldmain);
bfd = blo_read_file_internal(fd, filename);
/* ensures relinked images are not freed */
@ -319,6 +322,9 @@ BlendFileData *BLO_read_from_memfile(Main *oldmain, const char *filename, MemFil
/* ensures relinked movie clips are not freed */
blo_end_movieclip_pointer_map(fd, oldmain);
/* ensures relinked packed data is not freed */
blo_end_packed_pointer_map(fd, oldmain);
/* move libraries from old main to new main */
if (bfd && mainlist.first != mainlist.last) {

View File

@ -543,7 +543,9 @@ static Main *blo_find_main(FileData *fd, const char *filepath, const char *relab
BLI_strncpy(name1, filepath, sizeof(name1));
cleanup_path(relabase, name1);
// printf("blo_find_main: original in %s\n", name);
// printf("blo_find_main: relabase %s\n", relabase);
// printf("blo_find_main: original in %s\n", filepath);
// printf("blo_find_main: converted to %s\n", name1);
for (m = mainlist->first; m; m = m->next) {
@ -1020,6 +1022,46 @@ FileData *blo_openblenderfile(const char *filepath, ReportList *reports)
}
}
static int fd_read_gzip_from_memory(FileData *filedata, void *buffer, unsigned int size)
{
int err;
filedata->strm.next_out = (Bytef *) buffer;
filedata->strm.avail_out = size;
// Inflate another chunk.
err = inflate (&filedata->strm, Z_SYNC_FLUSH);
if (err == Z_STREAM_END) {
return 0;
}
else if (err != Z_OK) {
printf("fd_read_gzip_from_memory: zlib error\n");
return 0;
}
filedata->seek += size;
return (size);
}
static int fd_read_gzip_from_memory_init(FileData *fd)
{
fd->strm.next_in = (Bytef *) fd->buffer;
fd->strm.avail_in = fd->buffersize;
fd->strm.total_out = 0;
fd->strm.zalloc = Z_NULL;
fd->strm.zfree = Z_NULL;
if (inflateInit2(&fd->strm, (16+MAX_WBITS)) != Z_OK)
return 0;
fd->read = fd_read_gzip_from_memory;
return 1;
}
FileData *blo_openblendermemory(void *mem, int memsize, ReportList *reports)
{
if (!mem || memsize<SIZEOFBLENDERHEADER) {
@ -1028,11 +1070,23 @@ FileData *blo_openblendermemory(void *mem, int memsize, ReportList *reports)
}
else {
FileData *fd = filedata_new();
char *cp = mem;
fd->buffer = mem;
fd->buffersize = memsize;
fd->read = fd_read_from_memory;
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
/* test if gzip */
if (cp[0] == 0x1f && cp[1] == 0x8b) {
if (0 == fd_read_gzip_from_memory_init(fd)) {
blo_freefiledata(fd);
return NULL;
}
}
else
fd->read = fd_read_from_memory;
fd->flags |= FD_FLAGS_NOT_MY_BUFFER;
return blo_decode_and_check(fd, reports);
}
}
@ -1066,6 +1120,12 @@ void blo_freefiledata(FileData *fd)
gzclose(fd->gzfiledes);
}
if (fd->strm.next_in) {
if (inflateEnd (&fd->strm) != Z_OK) {
printf("close gzip stream error\n");
}
}
if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) {
MEM_freeN(fd->buffer);
fd->buffer = NULL;
@ -1154,25 +1214,33 @@ static void *newdataadr(FileData *fd, void *adr) /* only direct databocks */
return oldnewmap_lookup_and_inc(fd->datamap, adr);
}
static void *newglobadr(FileData *fd, void *adr) /* direct datablocks with global linking */
static void *newglobadr(FileData *fd, void *adr) /* direct datablocks with global linking */
{
return oldnewmap_lookup_and_inc(fd->globmap, adr);
}
static void *newimaadr(FileData *fd, void *adr) /* used to restore image data after undo */
static void *newimaadr(FileData *fd, void *adr) /* used to restore image data after undo */
{
if (fd->imamap && adr)
return oldnewmap_lookup_and_inc(fd->imamap, adr);
return NULL;
}
static void *newmclipadr(FileData *fd, void *adr) /* used to restore movie clip data after undo */
static void *newmclipadr(FileData *fd, void *adr) /* used to restore movie clip data after undo */
{
if (fd->movieclipmap && adr)
return oldnewmap_lookup_and_inc(fd->movieclipmap, adr);
return NULL;
}
static void *newpackedadr(FileData *fd, void *adr) /* used to restore packed data after undo */
{
if (fd->packedmap && adr)
return oldnewmap_lookup_and_inc(fd->packedmap, adr);
return oldnewmap_lookup_and_inc(fd->datamap, adr);
}
static void *newlibadr(FileData *fd, void *lib, void *adr) /* only lib data */
{
@ -1369,6 +1437,69 @@ void blo_end_movieclip_pointer_map(FileData *fd, Main *oldmain)
}
}
static void insert_packedmap(FileData *fd, PackedFile *pf)
{
oldnewmap_insert(fd->packedmap, pf, pf, 0);
oldnewmap_insert(fd->packedmap, pf->data, pf->data, 0);
}
void blo_make_packed_pointer_map(FileData *fd, Main *oldmain)
{
Image *ima;
VFont *vfont;
bSound *sound;
Library *lib;
fd->packedmap = oldnewmap_new();
for (ima = oldmain->image.first; ima; ima = ima->id.next)
if (ima->packedfile)
insert_packedmap(fd, ima->packedfile);
for (vfont = oldmain->vfont.first; vfont; vfont = vfont->id.next)
if (vfont->packedfile)
insert_packedmap(fd, vfont->packedfile);
for (sound = oldmain->sound.first; sound; sound = sound->id.next)
if (sound->packedfile)
insert_packedmap(fd, sound->packedfile);
for (lib = oldmain->library.first; lib; lib = lib->id.next)
if (lib->packedfile)
insert_packedmap(fd, lib->packedfile);
}
/* set old main packed data to zero if it has been restored */
/* this works because freeing old main only happens after this call */
void blo_end_packed_pointer_map(FileData *fd, Main *oldmain)
{
Image *ima;
VFont *vfont;
bSound *sound;
Library *lib;
OldNew *entry = fd->packedmap->entries;
int i;
/* used entries were restored, so we put them to zero */
for (i=0; i < fd->packedmap->nentries; i++, entry++) {
if (entry->nr > 0)
entry->newp = NULL;
}
for (ima = oldmain->image.first; ima; ima = ima->id.next)
ima->packedfile = newpackedadr(fd, ima->packedfile);
for (vfont = oldmain->vfont.first; vfont; vfont = vfont->id.next)
vfont->packedfile = newpackedadr(fd, vfont->packedfile);
for (sound = oldmain->sound.first; sound; sound = sound->id.next)
sound->packedfile = newpackedadr(fd, sound->packedfile);
for (lib = oldmain->library.first; lib; lib = lib->id.next)
lib->packedfile = newpackedadr(fd, lib->packedfile);
}
/* undo file support: add all library pointers in lookup */
void blo_add_library_pointer_map(ListBase *mainlist, FileData *fd)
@ -1708,10 +1839,10 @@ static void direct_link_script(FileData *UNUSED(fd), Script *script)
static PackedFile *direct_link_packedfile(FileData *fd, PackedFile *oldpf)
{
PackedFile *pf = newdataadr(fd, oldpf);
PackedFile *pf = newpackedadr(fd, oldpf);
if (pf) {
pf->data = newdataadr(fd, pf->data);
pf->data = newpackedadr(fd, pf->data);
}
return pf;
@ -6031,6 +6162,7 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main)
{
Main *newmain;
/* check if the library was already read */
for (newmain = fd->mainlist->first; newmain; newmain = newmain->next) {
if (newmain->curlib) {
if (BLI_path_cmp(newmain->curlib->filepath, lib->filepath) == 0) {
@ -6049,14 +6181,14 @@ static void direct_link_library(FileData *fd, Library *lib, Main *main)
}
}
}
/* make sure we have full path in lib->filename */
/* make sure we have full path in lib->filepath */
BLI_strncpy(lib->filepath, lib->name, sizeof(lib->name));
cleanup_path(fd->relabase, lib->filepath);
#if 0
printf("direct_link_library: name %s\n", lib->name);
printf("direct_link_library: filename %s\n", lib->filename);
#endif
// printf("direct_link_library: name %s\n", lib->name);
// printf("direct_link_library: filepath %s\n", lib->filepath);
lib->packedfile = direct_link_packedfile(fd, lib->packedfile);
/* new main */
newmain= MEM_callocN(sizeof(Main), "directlink");
@ -10014,12 +10146,26 @@ static void read_libraries(FileData *basefd, ListBase *mainlist)
FileData *fd = mainptr->curlib->filedata;
if (fd == NULL) {
/* printf and reports for now... its important users know this */
BKE_reportf_wrap(basefd->reports, RPT_INFO, TIP_("Read library: '%s', '%s'"),
mainptr->curlib->filepath, mainptr->curlib->name);
fd = blo_openblenderfile(mainptr->curlib->filepath, basefd->reports);
/* printf and reports for now... its important users know this */
/* if packed file... */
if (mainptr->curlib->packedfile) {
PackedFile *pf = mainptr->curlib->packedfile;
BKE_reportf_wrap(basefd->reports, RPT_INFO, TIP_("Read packed library: '%s'"),
mainptr->curlib->name);
fd = blo_openblendermemory(pf->data, pf->size, basefd->reports);
/* needed for library_append and read_libraries */
BLI_strncpy(fd->relabase, mainptr->curlib->filepath, sizeof(fd->relabase));
}
else {
BKE_reportf_wrap(basefd->reports, RPT_INFO, TIP_("Read library: '%s', '%s'"),
mainptr->curlib->filepath, mainptr->curlib->name);
fd = blo_openblenderfile(mainptr->curlib->filepath, basefd->reports);
}
/* allow typing in a new lib path */
if (G.debug_value == -666) {
while (fd == NULL) {

View File

@ -69,6 +69,9 @@ typedef struct FileData {
char headerdone;
int inbuffer;
// gzip stream for memory decompression
z_stream strm;
// general reading variables
struct SDNA *filesdna;
struct SDNA *memsdna;
@ -83,6 +86,7 @@ typedef struct FileData {
struct OldNewMap *libmap;
struct OldNewMap *imamap;
struct OldNewMap *movieclipmap;
struct OldNewMap *packedmap;
struct BHeadSort *bheadmap;
int tot_bheadmap;
@ -127,6 +131,8 @@ void blo_make_image_pointer_map(FileData *fd, Main *oldmain);
void blo_end_image_pointer_map(FileData *fd, Main *oldmain);
void blo_make_movieclip_pointer_map(FileData *fd, Main *oldmain);
void blo_end_movieclip_pointer_map(FileData *fd, Main *oldmain);
void blo_make_packed_pointer_map(FileData *fd, Main *oldmain);
void blo_end_packed_pointer_map(FileData *fd, Main *oldmain);
void blo_add_library_pointer_map(ListBase *mainlist, FileData *fd);
void blo_freefiledata(FileData *fd);

View File

@ -1508,7 +1508,7 @@ static void write_vfonts(WriteData *wd, ListBase *idbase)
/* direct data */
if (vf->packedfile) {
if (vf->packedfile && !wd->current) {
pf = vf->packedfile;
writestruct(wd, DATA, "PackedFile", 1, pf);
writedata(wd, DATA, pf->size, pf->data);
@ -1958,7 +1958,7 @@ static void write_images(WriteData *wd, ListBase *idbase)
writestruct(wd, ID_IM, "Image", 1, ima);
if (ima->id.properties) IDP_WriteProperty(ima->id.properties, wd);
if (ima->packedfile) {
if (ima->packedfile && !wd->current) {
pf = ima->packedfile;
writestruct(wd, DATA, "PackedFile", 1, pf);
writedata(wd, DATA, pf->size, pf->data);
@ -2531,20 +2531,31 @@ static void write_libraries(WriteData *wd, Main *main)
a=tot= set_listbasepointers(main, lbarray);
/* test: is lib being used */
foundone = FALSE;
while (tot--) {
for (id= lbarray[tot]->first; id; id= id->next) {
if (id->us>0 && (id->flag & LIB_EXTERN)) {
foundone = TRUE;
break;
if (main->curlib && main->curlib->packedfile)
foundone = TRUE;
else {
foundone = FALSE;
while (tot--) {
for (id= lbarray[tot]->first; id; id= id->next) {
if (id->us>0 && (id->flag & LIB_EXTERN)) {
foundone = TRUE;
break;
}
}
if (foundone) break;
}
if (foundone) break;
}
if (foundone) {
writestruct(wd, ID_LI, "Library", 1, main->curlib);
if (main->curlib->packedfile && !wd->current) {
PackedFile *pf = main->curlib->packedfile;
writestruct(wd, DATA, "PackedFile", 1, pf);
writedata(wd, DATA, pf->size, pf->data);
printf("write packed .blend: %s\n", main->curlib->name);
}
while (a--) {
for (id= lbarray[a]->first; id; id= id->next) {
if (id->us>0 && (id->flag & LIB_EXTERN)) {
@ -2673,7 +2684,7 @@ static void write_sounds(WriteData *wd, ListBase *idbase)
writestruct(wd, ID_SO, "bSound", 1, sound);
if (sound->id.properties) IDP_WriteProperty(sound->id.properties, wd);
if (sound->packedfile) {
if (sound->packedfile && !wd->current) {
pf = sound->packedfile;
writestruct(wd, DATA, "PackedFile", 1, pf);
writedata(wd, DATA, pf->size, pf->data);

View File

@ -39,11 +39,15 @@ struct ReportList;
void FILE_OT_pack_all(struct wmOperatorType *ot);
void FILE_OT_unpack_all(struct wmOperatorType *ot);
void FILE_OT_pack_libraries(struct wmOperatorType *ot);
void FILE_OT_unpack_libraries(struct wmOperatorType *ot);
void FILE_OT_make_paths_relative(struct wmOperatorType *ot);
void FILE_OT_make_paths_absolute(struct wmOperatorType *ot);
void FILE_OT_report_missing_files(struct wmOperatorType *ot);
void FILE_OT_find_missing_files(struct wmOperatorType *ot);
void INFO_OT_reports_display_update(struct wmOperatorType *ot);
/* info_draw.c */

View File

@ -66,15 +66,70 @@
#include "info_intern.h"
/********************* pack blend file libararies operator *********************/
static int pack_libraries_exec(bContext *C, wmOperator *op)
{
Main *bmain = CTX_data_main(C);
packLibraries(bmain, op->reports);
return OPERATOR_FINISHED;
}
void FILE_OT_pack_libraries(wmOperatorType *ot)
{
/* identifiers */
ot->name = "Pack Blender Libraries";
ot->idname = "FILE_OT_pack_libraries";
ot->description = "Pack all used Blender library files into the current .blend";
/* api callbacks */
ot->exec = pack_libraries_exec;
/* flags */
ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
}
static int unpack_libraries_exec(bContext *C, wmOperator *op)
{
Main *bmain = CTX_data_main(C);
unpackLibraries(bmain, op->reports);
return OPERATOR_FINISHED;
}
static int unpack_libraries_invoke(bContext *C, wmOperator *op, wmEvent *UNUSED(event))
{
return WM_operator_confirm_message(C, op, "Unpack Blender Libraries - creates directories, all new paths should work");
}
void FILE_OT_unpack_libraries(wmOperatorType *ot)
{
/* identifiers */
ot->name = "Unpack Blender Libraries";
ot->idname = "FILE_OT_unpack_libraries";
ot->description = "Unpack all used Blender library files from this .blend file";
/* api callbacks */
ot->invoke = unpack_libraries_invoke;
ot->exec = unpack_libraries_exec;
/* flags */
ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
}
/********************* pack all operator *********************/
static int pack_all_exec(bContext *C, wmOperator *op)
{
Main *bmain = CTX_data_main(C);
packAll(bmain, op->reports);
G.fileflags |= G_AUTOPACK;
return OPERATOR_FINISHED;
}
@ -83,7 +138,7 @@ static int pack_all_invoke(bContext *C, wmOperator *op, wmEvent *UNUSED(event))
Main *bmain = CTX_data_main(C);
Image *ima;
ImBuf *ibuf;
// first check for dirty images
for (ima = bmain->image.first; ima; ima = ima->id.next) {
if (ima->ibufs.first) { /* XXX FIX */
@ -93,16 +148,16 @@ static int pack_all_invoke(bContext *C, wmOperator *op, wmEvent *UNUSED(event))
BKE_image_release_ibuf(ima, ibuf, NULL);
break;
}
BKE_image_release_ibuf(ima, ibuf, NULL);
}
}
if (ima) {
uiPupMenuOkee(C, "FILE_OT_pack_all", "Some images are painted on. These changes will be lost. Continue?");
return OPERATOR_CANCELLED;
}
return pack_all_exec(C, op);
}
@ -116,11 +171,12 @@ void FILE_OT_pack_all(wmOperatorType *ot)
/* api callbacks */
ot->exec = pack_all_exec;
ot->invoke = pack_all_invoke;
/* flags */
ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
}
/********************* unpack all operator *********************/
static const EnumPropertyItem unpack_all_method_items[] = {

View File

@ -178,7 +178,10 @@ static void info_main_area_draw(const bContext *C, ARegion *ar)
static void info_operatortypes(void)
{
WM_operatortype_append(FILE_OT_pack_all);
WM_operatortype_append(FILE_OT_pack_libraries);
WM_operatortype_append(FILE_OT_unpack_all);
WM_operatortype_append(FILE_OT_unpack_libraries);
WM_operatortype_append(FILE_OT_make_paths_relative);
WM_operatortype_append(FILE_OT_make_paths_absolute);
WM_operatortype_append(FILE_OT_report_missing_files);

View File

@ -42,6 +42,7 @@ extern "C" {
struct Library;
struct FileData;
struct ID;
struct PackedFile;
typedef struct IDPropertyData {
void *pointer;
@ -136,6 +137,8 @@ typedef struct Library {
* setting 'name' directly and it will be kept in
* sync - campbell */
struct Library *parent; /* set for indirectly linked libs, used in the outliner and while reading */
struct PackedFile *packedfile;
} Library;
enum eIconSizes {