Index: user/uqs/git_conv/svn2git/.gitignore =================================================================== --- user/uqs/git_conv/svn2git/.gitignore (nonexistent) +++ user/uqs/git_conv/svn2git/.gitignore (revision 311976) @@ -0,0 +1,5 @@ +Makefile +*~ +*.o +svn-all-fast-export +src/local-config.pri Index: user/uqs/git_conv/svn2git/Makefile =================================================================== --- user/uqs/git_conv/svn2git/Makefile (nonexistent) +++ user/uqs/git_conv/svn2git/Makefile (revision 311976) @@ -0,0 +1,128 @@ +############################################################################# +# Makefile for building: fast-export2 +# Generated by qmake (2.01a) (Qt 4.8.7) on: Thu Jan 12 12:51:50 2017 +# Project: fast-export2.pro +# Template: subdirs +# Command: /usr/local/bin/qmake-qt4 -o Makefile fast-export2.pro +############################################################################# + +first: make_default +MAKEFILE = Makefile +QMAKE = /usr/local/bin/qmake-qt4 +DEL_FILE = rm -f +CHK_DIR_EXISTS= test -d +MKDIR = mkdir -p +COPY = cp -f +COPY_FILE = cp -f +COPY_DIR = cp -f -R +INSTALL_FILE = install -m 644 -p +INSTALL_PROGRAM = install -m 755 -p +INSTALL_DIR = $(COPY_DIR) +DEL_FILE = rm -f +SYMLINK = ln -f -s +DEL_DIR = rmdir +MOVE = mv -f +CHK_DIR_EXISTS= test -d +MKDIR = mkdir -p +SUBTARGETS = \ + sub-src + +src/$(MAKEFILE): + @$(CHK_DIR_EXISTS) src/ || $(MKDIR) src/ + cd src/ && $(QMAKE) /data/src/git_conv/svn2git/src/src.pro -o $(MAKEFILE) +sub-src-qmake_all: FORCE + @$(CHK_DIR_EXISTS) src/ || $(MKDIR) src/ + cd src/ && $(QMAKE) /data/src/git_conv/svn2git/src/src.pro -o $(MAKEFILE) +sub-src: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) +sub-src-make_default: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) +sub-src-make_first: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) first +sub-src-all: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) all +sub-src-clean: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) clean +sub-src-distclean: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) distclean +sub-src-install_subtargets: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) install +sub-src-uninstall_subtargets: src/$(MAKEFILE) FORCE + cd src/ && $(MAKE) -f $(MAKEFILE) uninstall + +Makefile: fast-export2.pro /usr/local/share/qt4/mkspecs/freebsd-clang/qmake.conf /usr/local/share/qt4/mkspecs/common/unix.conf \ + /usr/local/share/qt4/mkspecs/common/freebsd.conf \ + /usr/local/share/qt4/mkspecs/common/gcc-base.conf \ + /usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf \ + /usr/local/share/qt4/mkspecs/common/clang.conf \ + /usr/local/share/qt4/mkspecs/common/clang-unix.conf \ + /usr/local/share/qt4/mkspecs/qconfig.pri \ + /usr/local/share/qt4/mkspecs/features/qt_functions.prf \ + /usr/local/share/qt4/mkspecs/features/qt_config.prf \ + /usr/local/share/qt4/mkspecs/features/exclusive_builds.prf \ + /usr/local/share/qt4/mkspecs/features/default_pre.prf \ + /usr/local/share/qt4/mkspecs/features/release.prf \ + /usr/local/share/qt4/mkspecs/features/default_post.prf \ + /usr/local/share/qt4/mkspecs/features/shared.prf \ + /usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \ + /usr/local/share/qt4/mkspecs/features/warn_on.prf \ + /usr/local/share/qt4/mkspecs/features/qt.prf \ + /usr/local/share/qt4/mkspecs/features/unix/thread.prf \ + /usr/local/share/qt4/mkspecs/features/moc.prf \ + /usr/local/share/qt4/mkspecs/features/resources.prf \ + /usr/local/share/qt4/mkspecs/features/uic.prf \ + /usr/local/share/qt4/mkspecs/features/yacc.prf \ + /usr/local/share/qt4/mkspecs/features/lex.prf \ + /usr/local/share/qt4/mkspecs/features/include_source_dir.prf + $(QMAKE) -o Makefile fast-export2.pro +/usr/local/share/qt4/mkspecs/common/unix.conf: +/usr/local/share/qt4/mkspecs/common/freebsd.conf: +/usr/local/share/qt4/mkspecs/common/gcc-base.conf: +/usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf: +/usr/local/share/qt4/mkspecs/common/clang.conf: +/usr/local/share/qt4/mkspecs/common/clang-unix.conf: +/usr/local/share/qt4/mkspecs/qconfig.pri: +/usr/local/share/qt4/mkspecs/features/qt_functions.prf: +/usr/local/share/qt4/mkspecs/features/qt_config.prf: +/usr/local/share/qt4/mkspecs/features/exclusive_builds.prf: +/usr/local/share/qt4/mkspecs/features/default_pre.prf: +/usr/local/share/qt4/mkspecs/features/release.prf: +/usr/local/share/qt4/mkspecs/features/default_post.prf: +/usr/local/share/qt4/mkspecs/features/shared.prf: +/usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf: +/usr/local/share/qt4/mkspecs/features/warn_on.prf: +/usr/local/share/qt4/mkspecs/features/qt.prf: +/usr/local/share/qt4/mkspecs/features/unix/thread.prf: +/usr/local/share/qt4/mkspecs/features/moc.prf: +/usr/local/share/qt4/mkspecs/features/resources.prf: +/usr/local/share/qt4/mkspecs/features/uic.prf: +/usr/local/share/qt4/mkspecs/features/yacc.prf: +/usr/local/share/qt4/mkspecs/features/lex.prf: +/usr/local/share/qt4/mkspecs/features/include_source_dir.prf: +qmake: qmake_all FORCE + @$(QMAKE) -o Makefile fast-export2.pro + +qmake_all: sub-src-qmake_all FORCE + +make_default: sub-src-make_default FORCE +make_first: sub-src-make_first FORCE +all: sub-src-all FORCE +clean: sub-src-clean FORCE +distclean: sub-src-distclean FORCE + -$(DEL_FILE) Makefile +install_subtargets: sub-src-install_subtargets FORCE +uninstall_subtargets: sub-src-uninstall_subtargets FORCE + +sub-src-check: src/$(MAKEFILE) + cd src/ && $(MAKE) -f $(MAKEFILE) check +check: sub-src-check + +mocclean: compiler_moc_header_clean compiler_moc_source_clean + +mocables: compiler_moc_header_make_all compiler_moc_source_make_all +install: install_subtargets FORCE + +uninstall: uninstall_subtargets FORCE + +FORCE: + Property changes on: user/uqs/git_conv/svn2git/Makefile ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/uqs/git_conv/svn2git/README.md =================================================================== --- user/uqs/git_conv/svn2git/README.md (nonexistent) +++ user/uqs/git_conv/svn2git/README.md (revision 311976) @@ -0,0 +1,40 @@ +svn-all-fast-export aka svn2git +=============================== +This project contains all the tools required to do a conversion of an svn repository (server side, not a checkout) to one or more git repositories. + +This is the tool used to convert KDE's Subversion into multiple Git repositories. You can find more description and usage examples at https://techbase.kde.org/Projects/MoveToGit/UsingSvn2Git + + +How does it work +---------------- +The svn2git repository gets you an application that will do the actual conversion. +The conversion exists of looping over each and every commit in the subversion repository and matching the changes to a ruleset after which the changes are applied to a certain path in a git repo. +The ruleset can specify which git repository to use and thus you can have more than one git repository as a result of running the conversion. +Also noteworthy is that you can have a rule that, for example, changes in svnrepo/branches/foo/2.1/ will appear as a git-branch in a repository. + +If you have a proper ruleset the tool will create the git repositories for you and show progress while converting commit by commit. + +After it is done you likely want to run `git repack -a -d -f` to compress the pack file as it can get quite big. + +Building the tool +----------------- +Run `qmake && make`. You get `./svn-all-fast-export`. +(Do a checkout of the repo .git' and run qmake and make. You can only build it after having installed libsvn-dev, and naturally Qt. Running the command will give you all the options you can pass to the tool.) + +KDE +--- +there is a repository kde-ruleset which has several example files and one file that should become the final ruleset for the whole of KDE called 'kde-rules-main'. + +Write the Rules +--------------- +You need to write a rules file that describes how to slice the Subversion history into Git repositories and branches. See https://techbase.kde.org/Projects/MoveToGit/UsingSvn2Git. +The rules are also documented in the 'samples' directory of the svn2git repository. Feel free to add more documentation here as well. + +Work flow +--------- +Please feel free to fill this section in. + +Some SVN tricks +--------------- +You can access your newly rsynced SVN repo with commands like `svn ls file:///path/to/repo/trunk/KDE`. +A common issue is tracking when an item left playground for kdereview and then went from kdereview to its final destination. There is no straightforward way to do this. So the following command comes in handy: `svn log -v file:///path/to/repo/kde-svn/kde/trunk/kdereview | grep /trunk/kdereview/mplayerthumbs -A 5 -B 5` This will print all commits relevant to the package you are trying to track. You can also pipe the above command to head or tail to see the the first and last commit it was in that directory. Index: user/uqs/git_conv/svn2git/samples/ignore-branch.rules =================================================================== --- user/uqs/git_conv/svn2git/samples/ignore-branch.rules (nonexistent) +++ user/uqs/git_conv/svn2git/samples/ignore-branch.rules (revision 311976) @@ -0,0 +1,31 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/ + repository myproject + branch master +end match + +# Ignore this branch: +# We ignore a branch by not telling svn-all-fast-export what to do +# with this path +# Note that rules are applied in order of appearance, so this rule +# must appear before the generic branch rule +match /branches/abandoned-work/ +end match + +match /branches/([^/]+)/ + repository myproject + branch \1 +end match + +# No tag processing Index: user/uqs/git_conv/svn2git/samples/merged-branches-tags.rules =================================================================== --- user/uqs/git_conv/svn2git/samples/merged-branches-tags.rules (nonexistent) +++ user/uqs/git_conv/svn2git/samples/merged-branches-tags.rules (revision 311976) @@ -0,0 +1,31 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/ + repository myproject + branch master +end match + +# Subversion doesn't understand the Git concept of tags +# In Subversion, tags are really branches +# +# Only a post-processing (i.e., after converting to Git) of the tag +# branches can we be sure that a tag wasn't moved or changed from the +# branch it was copied from +# +# So we don't pretend that SVN tags are Git tags and then import +# everything as one + +match /(branches|tags)/([^/]+)/ + repository myproject + branch \2 +end match Index: user/uqs/git_conv/svn2git/samples/min-max-revision.rules =================================================================== --- user/uqs/git_conv/svn2git/samples/min-max-revision.rules (nonexistent) +++ user/uqs/git_conv/svn2git/samples/min-max-revision.rules (revision 311976) @@ -0,0 +1,39 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +# Ignore this particular revision in trunk +# See ignore-branch.rules first +# Note that rules are applied in order of appearance, so this rule +# must appear before the generic trunk rule +match /trunk/ + min revision 123 + max revision 123 +end match + +# Stop importing trunk +# If we don't specify a max revision, then there is no limit +# The same applies to min revision (i.e., min revision is 0) +match /trunk/ + min revision 1234 +end match + +match /trunk/ + repository myproject + branch master +end match + +match /branches/([^/]+)/ + repository myproject + branch \1 +end match + +# No tag processing Index: user/uqs/git_conv/svn2git/samples/recurse.rules =================================================================== --- user/uqs/git_conv/svn2git/samples/recurse.rules (nonexistent) +++ user/uqs/git_conv/svn2git/samples/recurse.rules (revision 311976) @@ -0,0 +1,41 @@ +# +# Declare the repositories we know about: +# + +create repository project1 +end repository + +create repository project2 +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/([^/]+)/ + repository \1 + branch master +end match + +# +# SVN layout: +# /branches/branchname/project1 +# /branches/branchname/project2 +match /branches/([^/]+)/([^/]+)/ + repository \2 + branch \1 +end match + + +# +# Example of the recurse rule: +# We tell svn-all-fast-export to not import anything +# but to go inside and recurse in the subdirs +# Note how the ending slash is missing in this particular case + +match /branches/[^/]+ + action recurse +end match + +# No tag processing Index: user/uqs/git_conv/svn2git/samples/standardlayout.rules =================================================================== --- user/uqs/git_conv/svn2git/samples/standardlayout.rules (nonexistent) +++ user/uqs/git_conv/svn2git/samples/standardlayout.rules (revision 311976) @@ -0,0 +1,37 @@ +# +# Declare the repositories we know about: +# + +create repository myproject +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /trunk/ + repository myproject + branch master +end match + +match /branches/([^/]+)/ + repository myproject + branch \1 +end match + +# Important: +# Subversion doesn't understand the Git concept of tags +# In Subversion, tags are really branches +# +# Only a post-processing (i.e., after converting to Git) of the tag +# branches can we be sure that a tag wasn't moved or changed from the +# branch it was copied from +# +# This rule will create tags that don't exist in any of the +# branches. It's not what you want. +# See the merged-branches-tags.rules file +match /tags/([^/]+)/ + repository myproject + branch refs/tags/\1 +end match Index: user/uqs/git_conv/svn2git/samples/two-projects.rules =================================================================== --- user/uqs/git_conv/svn2git/samples/two-projects.rules (nonexistent) +++ user/uqs/git_conv/svn2git/samples/two-projects.rules (revision 311976) @@ -0,0 +1,32 @@ +# +# Declare the repositories we know about: +# + +create repository project1 +end repository + +create repository project2 +end repository + +# +# Declare the rules +# Note: rules must end in a slash +# + +match /project1/trunk/ + repository project1 + branch master +end match + +match /project2/trunk/ + repository project2 + branch master +end match + +# Note how we can use regexp to capture the repository name +match /([^/]+)/branches/([^/]+)/ + repository \1 + branch \2 +end match + +# No tag processing Index: user/uqs/git_conv/svn2git/src/Makefile =================================================================== --- user/uqs/git_conv/svn2git/src/Makefile (revision 311975) +++ user/uqs/git_conv/svn2git/src/Makefile (revision 311976) @@ -1,242 +1,242 @@ ############################################################################# # Makefile for building: ../svn-all-fast-export -# Generated by qmake (2.01a) (Qt 4.8.7) on: Tue Nov 10 14:23:11 2015 +# Generated by qmake (2.01a) (Qt 4.8.7) on: Tue Jan 10 11:40:19 2017 # Project: src.pro # Template: app # Command: /usr/local/bin/qmake-qt4 -o Makefile src.pro ############################################################################# ####### Compiler, tools and options CC = clang CXX = clang++ -DEFINES = -DVER="\"fa63382f9549c6ae1df7b8880d9cced531ab4e98\"" -DQT_NO_DEBUG -DQT_CORE_LIB -DQT_SHARED +DEFINES = -DVER="\"437beb493db3b1cb532ca5807264cd4403a44157\"" -DQT_NO_DEBUG -DQT_CORE_LIB -DQT_SHARED CFLAGS = -pipe -O2 -Wall -W -pthread -D_THREAD_SAFE $(DEFINES) CXXFLAGS = -pipe -O2 -Wall -W -pthread -D_THREAD_SAFE $(DEFINES) INCPATH = -I/usr/local/share/qt4/mkspecs/freebsd-clang -I. -I/usr/local/include/qt4/QtCore -I/usr/local/include/qt4 -I. -I/usr/include/subversion-1 -I/usr/local/include/subversion-1 -I/usr/include/apr-1.0 -I/usr/include/apr-1 -I/usr/local/include/apr-1 -I. -I/usr/local/include/qt4 -I/usr/local/include LINK = clang++ LFLAGS = -Wl,-O1 -pthread -Wl,-rpath,/usr/local/lib/qt4 LIBS = $(SUBLIBS) -L/usr/local/lib/qt4 -L/usr/local/lib -lsvn_fs-1 -lsvn_repos-1 -lapr-1 -lsvn_subr-1 -lQtCore -L/usr/local/lib/qt4 -L/usr/local/lib AR = ar cqs RANLIB = QMAKE = /usr/local/bin/qmake-qt4 TAR = tar -cf COMPRESS = gzip -9f COPY = cp -f SED = sed COPY_FILE = cp -f COPY_DIR = cp -f -R STRIP = strip INSTALL_FILE = install -m 644 -p INSTALL_DIR = $(COPY_DIR) INSTALL_PROGRAM = install -m 755 -p DEL_FILE = rm -f SYMLINK = ln -f -s DEL_DIR = rmdir MOVE = mv -f CHK_DIR_EXISTS= test -d MKDIR = mkdir -p ####### Output directory OBJECTS_DIR = ./ ####### Files SOURCES = ruleparser.cpp \ repository.cpp \ svn.cpp \ main.cpp \ CommandLineParser.cpp OBJECTS = ruleparser.o \ repository.o \ svn.o \ main.o \ CommandLineParser.o DIST = /usr/local/share/qt4/mkspecs/common/unix.conf \ /usr/local/share/qt4/mkspecs/common/freebsd.conf \ /usr/local/share/qt4/mkspecs/common/gcc-base.conf \ /usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf \ /usr/local/share/qt4/mkspecs/common/clang.conf \ /usr/local/share/qt4/mkspecs/common/clang-unix.conf \ /usr/local/share/qt4/mkspecs/qconfig.pri \ /usr/local/share/qt4/mkspecs/features/qt_functions.prf \ /usr/local/share/qt4/mkspecs/features/qt_config.prf \ /usr/local/share/qt4/mkspecs/features/exclusive_builds.prf \ /usr/local/share/qt4/mkspecs/features/default_pre.prf \ /usr/local/share/qt4/mkspecs/features/release.prf \ /usr/local/share/qt4/mkspecs/features/default_post.prf \ /usr/local/share/qt4/mkspecs/features/shared.prf \ /usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \ /usr/local/share/qt4/mkspecs/features/warn_on.prf \ /usr/local/share/qt4/mkspecs/features/qt.prf \ /usr/local/share/qt4/mkspecs/features/unix/thread.prf \ /usr/local/share/qt4/mkspecs/features/moc.prf \ /usr/local/share/qt4/mkspecs/features/resources.prf \ /usr/local/share/qt4/mkspecs/features/uic.prf \ /usr/local/share/qt4/mkspecs/features/yacc.prf \ /usr/local/share/qt4/mkspecs/features/lex.prf \ /usr/local/share/qt4/mkspecs/features/include_source_dir.prf \ src.pro QMAKE_TARGET = svn-all-fast-export DESTDIR = ../ TARGET = ../svn-all-fast-export first: all ####### Implicit rules .SUFFIXES: .o .c .cpp .cc .cxx .C .cpp.o: $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" .cc.o: $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" .cxx.o: $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" .C.o: $(CXX) -c $(CXXFLAGS) $(INCPATH) -o "$@" "$<" .c.o: $(CC) -c $(CFLAGS) $(INCPATH) -o "$@" "$<" ####### Build rules all: Makefile $(TARGET) $(TARGET): $(OBJECTS) @$(CHK_DIR_EXISTS) ../ || $(MKDIR) ../ $(LINK) $(LFLAGS) -o $(TARGET) $(OBJECTS) $(OBJCOMP) $(LIBS) Makefile: src.pro /usr/local/share/qt4/mkspecs/freebsd-clang/qmake.conf /usr/local/share/qt4/mkspecs/common/unix.conf \ /usr/local/share/qt4/mkspecs/common/freebsd.conf \ /usr/local/share/qt4/mkspecs/common/gcc-base.conf \ /usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf \ /usr/local/share/qt4/mkspecs/common/clang.conf \ /usr/local/share/qt4/mkspecs/common/clang-unix.conf \ /usr/local/share/qt4/mkspecs/qconfig.pri \ /usr/local/share/qt4/mkspecs/features/qt_functions.prf \ /usr/local/share/qt4/mkspecs/features/qt_config.prf \ /usr/local/share/qt4/mkspecs/features/exclusive_builds.prf \ /usr/local/share/qt4/mkspecs/features/default_pre.prf \ /usr/local/share/qt4/mkspecs/features/release.prf \ /usr/local/share/qt4/mkspecs/features/default_post.prf \ /usr/local/share/qt4/mkspecs/features/shared.prf \ /usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf \ /usr/local/share/qt4/mkspecs/features/warn_on.prf \ /usr/local/share/qt4/mkspecs/features/qt.prf \ /usr/local/share/qt4/mkspecs/features/unix/thread.prf \ /usr/local/share/qt4/mkspecs/features/moc.prf \ /usr/local/share/qt4/mkspecs/features/resources.prf \ /usr/local/share/qt4/mkspecs/features/uic.prf \ /usr/local/share/qt4/mkspecs/features/yacc.prf \ /usr/local/share/qt4/mkspecs/features/lex.prf \ /usr/local/share/qt4/mkspecs/features/include_source_dir.prf \ /usr/local/lib/qt4/libQtCore.prl $(QMAKE) -o Makefile src.pro /usr/local/share/qt4/mkspecs/common/unix.conf: /usr/local/share/qt4/mkspecs/common/freebsd.conf: /usr/local/share/qt4/mkspecs/common/gcc-base.conf: /usr/local/share/qt4/mkspecs/common/gcc-base-unix.conf: /usr/local/share/qt4/mkspecs/common/clang.conf: /usr/local/share/qt4/mkspecs/common/clang-unix.conf: /usr/local/share/qt4/mkspecs/qconfig.pri: /usr/local/share/qt4/mkspecs/features/qt_functions.prf: /usr/local/share/qt4/mkspecs/features/qt_config.prf: /usr/local/share/qt4/mkspecs/features/exclusive_builds.prf: /usr/local/share/qt4/mkspecs/features/default_pre.prf: /usr/local/share/qt4/mkspecs/features/release.prf: /usr/local/share/qt4/mkspecs/features/default_post.prf: /usr/local/share/qt4/mkspecs/features/shared.prf: /usr/local/share/qt4/mkspecs/features/unix/gdb_dwarf_index.prf: /usr/local/share/qt4/mkspecs/features/warn_on.prf: /usr/local/share/qt4/mkspecs/features/qt.prf: /usr/local/share/qt4/mkspecs/features/unix/thread.prf: /usr/local/share/qt4/mkspecs/features/moc.prf: /usr/local/share/qt4/mkspecs/features/resources.prf: /usr/local/share/qt4/mkspecs/features/uic.prf: /usr/local/share/qt4/mkspecs/features/yacc.prf: /usr/local/share/qt4/mkspecs/features/lex.prf: /usr/local/share/qt4/mkspecs/features/include_source_dir.prf: /usr/local/lib/qt4/libQtCore.prl: qmake: FORCE @$(QMAKE) -o Makefile src.pro dist: - @$(CHK_DIR_EXISTS) .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 || $(MKDIR) .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 - $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98/ && $(COPY_FILE) --parents ruleparser.h repository.h svn.h CommandLineParser.h .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98/ && $(COPY_FILE) --parents ruleparser.cpp repository.cpp svn.cpp main.cpp CommandLineParser.cpp .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98/ && (cd `dirname .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98` && $(TAR) svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98.tar svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 && $(COMPRESS) svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98.tar) && $(MOVE) `dirname .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98`/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98.tar.gz . && $(DEL_FILE) -r .tmp/svn-all-fast-exportfa63382f9549c6ae1df7b8880d9cced531ab4e98 + @$(CHK_DIR_EXISTS) .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 || $(MKDIR) .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 + $(COPY_FILE) --parents $(SOURCES) $(DIST) .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157/ && $(COPY_FILE) --parents ruleparser.h repository.h svn.h CommandLineParser.h .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157/ && $(COPY_FILE) --parents ruleparser.cpp repository.cpp svn.cpp main.cpp CommandLineParser.cpp .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157/ && (cd `dirname .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157` && $(TAR) svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157.tar svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 && $(COMPRESS) svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157.tar) && $(MOVE) `dirname .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157`/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157.tar.gz . && $(DEL_FILE) -r .tmp/svn-all-fast-export437beb493db3b1cb532ca5807264cd4403a44157 clean:compiler_clean -$(DEL_FILE) $(OBJECTS) -$(DEL_FILE) *~ core *.core ####### Sub-libraries distclean: clean -$(DEL_FILE) $(TARGET) -$(DEL_FILE) Makefile check: first mocclean: compiler_moc_header_clean compiler_moc_source_clean mocables: compiler_moc_header_make_all compiler_moc_source_make_all compiler_moc_header_make_all: compiler_moc_header_clean: compiler_rcc_make_all: compiler_rcc_clean: compiler_image_collection_make_all: qmake_image_collection.cpp compiler_image_collection_clean: -$(DEL_FILE) qmake_image_collection.cpp compiler_moc_source_make_all: compiler_moc_source_clean: compiler_uic_make_all: compiler_uic_clean: compiler_yacc_decl_make_all: compiler_yacc_decl_clean: compiler_yacc_impl_make_all: compiler_yacc_impl_clean: compiler_lex_make_all: compiler_lex_clean: compiler_clean: ####### Compile ruleparser.o: ruleparser.cpp ruleparser.h \ CommandLineParser.h $(CXX) -c $(CXXFLAGS) $(INCPATH) -o ruleparser.o ruleparser.cpp repository.o: repository.cpp repository.h \ ruleparser.h \ CommandLineParser.h $(CXX) -c $(CXXFLAGS) $(INCPATH) -o repository.o repository.cpp svn.o: svn.cpp svn.h \ ruleparser.h \ CommandLineParser.h \ repository.h $(CXX) -c $(CXXFLAGS) $(INCPATH) -o svn.o svn.cpp main.o: main.cpp CommandLineParser.h \ ruleparser.h \ repository.h \ svn.h $(CXX) -c $(CXXFLAGS) $(INCPATH) -o main.o main.cpp CommandLineParser.o: CommandLineParser.cpp CommandLineParser.h $(CXX) -c $(CXXFLAGS) $(INCPATH) -o CommandLineParser.o CommandLineParser.cpp ####### Install install: FORCE uninstall: FORCE FORCE: Index: user/uqs/git_conv/svn2git/src/main.cpp =================================================================== --- user/uqs/git_conv/svn2git/src/main.cpp (revision 311975) +++ user/uqs/git_conv/svn2git/src/main.cpp (revision 311976) @@ -1,283 +1,288 @@ /* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include #include #include #include #include #include #include #include "CommandLineParser.h" #include "ruleparser.h" #include "repository.h" #include "svn.h" QHash loadIdentityMapFile(const QString &fileName) { QHash result; if (fileName.isEmpty()) return result; QFile file(fileName); if (!file.open(QIODevice::ReadOnly)) { fprintf(stderr, "Could not open file %s: %s", qPrintable(fileName), qPrintable(file.errorString())); return result; } while (!file.atEnd()) { QByteArray line = file.readLine(); int comment_pos = line.indexOf('#'); if (comment_pos != -1) line.truncate(comment_pos); line = line.trimmed(); int space = line.indexOf(' '); if (space == -1) continue; // invalid line // Support git-svn author files, too // - svn2git native: loginname Joe User // - git-svn: loginname = Joe User int rightspace = line.indexOf(" = "); int leftspace = space; if (rightspace == -1) { rightspace = space; } else { leftspace = rightspace; rightspace += 2; } QByteArray realname = line.mid(rightspace).trimmed(); line.truncate(leftspace); result.insert(line, realname); }; file.close(); return result; } QSet loadRevisionsFile( const QString &fileName, Svn &svn ) { QRegExp revint("(\\d+)\\s*(?:-\\s*(\\d+|HEAD))?"); QSet revisions; if(fileName.isEmpty()) return revisions; QFile file(fileName); if( !file.open(QIODevice::ReadOnly)) { fprintf(stderr, "Could not open file %s: %s\n", qPrintable(fileName), qPrintable(file.errorString())); return revisions; } bool ok; while(!file.atEnd()) { QByteArray line = file.readLine().trimmed(); revint.indexIn(line); if( revint.cap(2).isEmpty() ) { int rev = revint.cap(1).toInt(&ok); if(ok) { revisions.insert(rev); } else { fprintf(stderr, "Unable to convert %s to int, skipping revision.\n", qPrintable(QString(line))); } } else if( revint.captureCount() == 2 ) { int rev = revint.cap(1).toInt(&ok); if(!ok) { fprintf(stderr, "Unable to convert %s (%s) to int, skipping revisions.\n", qPrintable(revint.cap(1)), qPrintable(QString(line))); continue; } int lastrev = 0; if(revint.cap(2) == "HEAD") { lastrev = svn.youngestRevision(); ok = true; } else { lastrev = revint.cap(2).toInt(&ok); } if(!ok) { fprintf(stderr, "Unable to convert %s (%s) to int, skipping revisions.\n", qPrintable(revint.cap(2)), qPrintable(QString(line))); continue; } for(; rev <= lastrev; ++rev ) revisions.insert(rev); } else { fprintf(stderr, "Unable to convert %s to int, skipping revision.\n", qPrintable(QString(line))); } } file.close(); return revisions; } static const CommandLineOption options[] = { {"--identity-map FILENAME", "provide map between svn username and email"}, {"--identity-domain DOMAIN", "provide user domain if no map was given"}, {"--revisions-file FILENAME", "provide a file with revision number that should be processed"}, {"--rules FILENAME[,FILENAME]", "the rules file(s) that determines what goes where"}, + {"--msg-filter FILENAME", "External program / script to modify svn log message"}, {"--add-metadata", "if passed, each git commit will have svn commit info"}, {"--add-metadata-notes", "if passed, each git commit will have notes with svn commit info"}, {"--resume-from revision", "start importing at svn revision number"}, {"--max-rev revision", "stop importing at svn revision number"}, {"--dry-run", "don't actually write anything"}, + {"--create-dump", "don't create the repository but a dump file suitable for piping into fast-import"}, {"--debug-rules", "print what rule is being used for each file"}, {"--commit-interval NUMBER", "if passed the cache will be flushed to git every NUMBER of commits"}, {"--stats", "after a run print some statistics about the rules"}, {"--svn-branches", "Use the contents of SVN when creating branches, Note: SVN tags are branches as well"}, + {"--empty-dirs", "Add .gitignore-file for empty dirs"}, + {"--svn-ignore", "Import svn-ignore-properties via .gitignore"}, + {"--propcheck", "Check for svn-properties except svn-ignore"}, {"-h, --help", "show help"}, {"-v, --version", "show version"}, CommandLineLastOption }; int main(int argc, char **argv) { printf("Invoked as:'"); for(int i = 0; i < argc; ++i) printf(" %s", argv[i]); printf("'\n"); CommandLineParser::init(argc, argv); CommandLineParser::addOptionDefinitions(options); Stats::init(); CommandLineParser *args = CommandLineParser::instance(); if(args->contains(QLatin1String("version"))) { printf("Git version: %s\n", VER); return 0; } if (args->contains(QLatin1String("help")) || args->arguments().count() != 1) { args->usage(QString(), "[Path to subversion repo]"); return 0; } if (args->undefinedOptions().count()) { QTextStream out(stderr); out << "svn-all-fast-export failed: "; bool first = true; foreach (QString option, args->undefinedOptions()) { if (!first) out << " : "; out << "unrecognized option or missing argument for; `" << option << "'" << endl; first = false; } return 10; } if (!args->contains("rules")) { QTextStream out(stderr); out << "svn-all-fast-export failed: please specify the rules using the 'rules' argument\n"; return 11; } if (!args->contains("identity-map") && !args->contains("identity-domain")) { QTextStream out(stderr); out << "WARNING; no identity-map or -domain specified, all commits will use default @localhost email address\n\n"; } QCoreApplication app(argc, argv); // Load the configuration RulesList rulesList(args->optionArgument(QLatin1String("rules"))); rulesList.load(); int resume_from = args->optionArgument(QLatin1String("resume-from")).toInt(); int max_rev = args->optionArgument(QLatin1String("max-rev")).toInt(); // create the repository list QHash repositories; int cutoff = resume_from ? resume_from : INT_MAX; retry: int min_rev = 1; foreach (Rules::Repository rule, rulesList.allRepositories()) { - Repository *repo = new Repository(rule); + Repository *repo = createRepository(rule, repositories); if (!repo) return EXIT_FAILURE; repositories.insert(rule.name, repo); int repo_next = repo->setupIncremental(cutoff); /* * cutoff < resume_from => error exit eventually * repo_next == cutoff => probably truncated log */ if (cutoff < resume_from && repo_next == cutoff) /* * Restore the log file so we fail the next time * svn2git is invoked with the same arguments */ repo->restoreLog(); if (cutoff < min_rev) /* * We've rewound before the last revision of some * repository that we've already seen. Start over * from the beginning. (since cutoff is decreasing, * we're sure we'll make forward progress eventually) */ goto retry; if (min_rev < repo_next) min_rev = repo_next; } if (cutoff < resume_from) { qCritical() << "Cannot resume from" << resume_from << "as there are errors in revision" << cutoff; return EXIT_FAILURE; } if (min_rev < resume_from) qDebug() << "skipping revisions" << min_rev << "to" << resume_from - 1 << "as requested"; if (resume_from) min_rev = resume_from; Svn::initialize(); Svn svn(args->arguments().first()); svn.setMatchRules(rulesList.allMatchRules()); svn.setRepositories(repositories); svn.setIdentityMap(loadIdentityMapFile(args->optionArgument("identity-map"))); // Massage user input a little, no guarantees that input makes sense. QString domain = args->optionArgument("identity-domain").simplified().remove(QChar('@')); if (domain.isEmpty()) domain = QString("localhost"); svn.setIdentityDomain(domain); if (max_rev < 1) max_rev = svn.youngestRevision(); bool errors = false; QSet revisions = loadRevisionsFile(args->optionArgument(QLatin1String("revisions-file")), svn); const bool filerRevisions = !revisions.isEmpty(); for (int i = min_rev; i <= max_rev; ++i) { if(filerRevisions) { if( !revisions.contains(i) ) { printf("."); continue; } else { printf("\n"); } } if (!svn.exportRevision(i)) { errors = true; break; } } foreach (Repository *repo, repositories) { repo->finalizeTags(); delete repo; } Stats::instance()->printStats(); return errors ? EXIT_FAILURE : EXIT_SUCCESS; } Index: user/uqs/git_conv/svn2git/src/repository.cpp =================================================================== --- user/uqs/git_conv/svn2git/src/repository.cpp (revision 311975) +++ user/uqs/git_conv/svn2git/src/repository.cpp (revision 311976) @@ -1,802 +1,1051 @@ /* * Copyright (C) 2007 Thiago Macieira * Copyright (C) 2009 Thomas Zander * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "repository.h" #include "CommandLineParser.h" #include #include #include #include #include static const int maxSimultaneousProcesses = 100; -static const int maxMark = (1 << 20) - 2; // some versions of git-fast-import are buggy for larger values of maxMark +typedef unsigned long long mark_t; +static const mark_t maxMark = ULONG_MAX; -class ProcessCache: QLinkedList +class FastImportRepository : public Repository { public: - void touch(Repository *repo) + class Transaction : public Repository::Transaction { + Q_DISABLE_COPY(Transaction) + friend class FastImportRepository; + + FastImportRepository *repository; + QByteArray branch; + QByteArray svnprefix; + QByteArray author; + QByteArray log; + uint datetime; + int revnum; + + QVector merges; + + QStringList deletedFiles; + QByteArray modifiedFiles; + + inline Transaction() {} + public: + ~Transaction(); + void commit(); + + void setAuthor(const QByteArray &author); + void setDateTime(uint dt); + void setLog(const QByteArray &log); + + void noteCopyFromBranch (const QString &prevbranch, int revFrom); + + void deleteFile(const QString &path); + QIODevice *addFile(const QString &path, int mode, qint64 length); + + void commitNote(const QByteArray ¬eText, bool append, + const QByteArray &commit = QByteArray()); + }; + FastImportRepository(const Rules::Repository &rule); + int setupIncremental(int &cutoff); + void restoreLog(); + ~FastImportRepository(); + + void reloadBranches(); + int createBranch(const QString &branch, int revnum, + const QString &branchFrom, int revFrom); + int deleteBranch(const QString &branch, int revnum); + Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum); + + void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, + const QByteArray &author, uint dt, + const QByteArray &log); + void finalizeTags(); + void commit(); + + bool branchExists(const QString& branch) const; + const QByteArray branchNote(const QString& branch) const; + void setBranchNote(const QString& branch, const QByteArray& noteText); + + bool hasPrefix() const; + + QString getName() const; + Repository *getEffectiveRepository(); +private: + struct Branch + { + int created; + QVector commits; + QVector marks; + QByteArray note; + }; + struct AnnotatedTag + { + QString supportingRef; + QByteArray svnprefix; + QByteArray author; + QByteArray log; + uint dt; + int revnum; + }; + + QHash branches; + QHash annotatedTags; + QString name; + QString prefix; + LoggingQProcess fastImport; + int commitCount; + int outstandingTransactions; + QByteArray deletedBranches; + QByteArray resetBranches; + + /* Optional filter to fix up log messages */ + QProcess filterMsg; + QByteArray msgFilter(QByteArray); + + /* starts at 0, and counts up. */ + mark_t last_commit_mark; + + /* starts at maxMark and counts down. Reset after each SVN revision */ + mark_t next_file_mark; + + bool processHasStarted; + + void startFastImport(); + void closeFastImport(); + + // called when a transaction is deleted + void forgetTransaction(Transaction *t); + + int resetBranch(const QString &branch, int revnum, mark_t mark, const QByteArray &resetTo, const QByteArray &comment); + long long markFrom(const QString &branchFrom, int branchRevNum, QByteArray &desc); + + friend class ProcessCache; + Q_DISABLE_COPY(FastImportRepository) +}; + +class ForwardingRepository : public Repository +{ + QString name; + Repository *repo; + QString prefix; +public: + class Transaction : public Repository::Transaction + { + Q_DISABLE_COPY(Transaction) + + Repository::Transaction *txn; + QString prefix; + public: + Transaction(Repository::Transaction *t, const QString &p) : txn(t), prefix(p) {} + ~Transaction() { delete txn; } + void commit() { txn->commit(); } + + void setAuthor(const QByteArray &author) { txn->setAuthor(author); } + void setDateTime(uint dt) { txn->setDateTime(dt); } + void setLog(const QByteArray &log) { txn->setLog(log); } + + void noteCopyFromBranch (const QString &prevbranch, int revFrom) + { txn->noteCopyFromBranch(prevbranch, revFrom); } + + void deleteFile(const QString &path) { txn->deleteFile(prefix + path); } + QIODevice *addFile(const QString &path, int mode, qint64 length) + { return txn->addFile(prefix + path, mode, length); } + + void commitNote(const QByteArray ¬eText, bool append, + const QByteArray &commit) + { return txn->commitNote(noteText, append, commit); } + }; + + ForwardingRepository(const QString &n, Repository *r, const QString &p) : name(n), repo(r), prefix(p) {} + + int setupIncremental(int &) { return 1; } + void restoreLog() {} + + void reloadBranches() { return repo->reloadBranches(); } + int createBranch(const QString &branch, int revnum, + const QString &branchFrom, int revFrom) + { return repo->createBranch(branch, revnum, branchFrom, revFrom); } + + int deleteBranch(const QString &branch, int revnum) + { return repo->deleteBranch(branch, revnum); } + + Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum) + { + Repository::Transaction *t = repo->newTransaction(branch, svnprefix, revnum); + return new Transaction(t, prefix); + } + + void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, + const QByteArray &author, uint dt, + const QByteArray &log) + { repo->createAnnotatedTag(name, svnprefix, revnum, author, dt, log); } + void finalizeTags() { /* loop that called this will invoke it on 'repo' too */ } + void commit() { repo->commit(); } + + bool branchExists(const QString& branch) const + { return repo->branchExists(branch); } + const QByteArray branchNote(const QString& branch) const + { return repo->branchNote(branch); } + void setBranchNote(const QString& branch, const QByteArray& noteText) + { repo->setBranchNote(branch, noteText); } + + bool hasPrefix() const + { return !prefix.isEmpty() || repo->hasPrefix(); } + + QString getName() const + { return name; } + Repository *getEffectiveRepository() + { return repo->getEffectiveRepository(); } +}; + +class ProcessCache: QLinkedList +{ +public: + void touch(FastImportRepository *repo) + { remove(repo); // if the cache is too big, remove from the front while (size() >= maxSimultaneousProcesses) takeFirst()->closeFastImport(); // append to the end append(repo); } - inline void remove(Repository *repo) + inline void remove(FastImportRepository *repo) { #if QT_VERSION >= 0x040400 removeOne(repo); #else removeAll(repo); #endif } }; static ProcessCache processCache; +Repository *createRepository(const Rules::Repository &rule, const QHash &repositories) +{ + if (rule.forwardTo.isEmpty()) + return new FastImportRepository(rule); + Repository *r = repositories[rule.forwardTo]; + if (!r) { + qCritical() << "no repository with name" << rule.forwardTo << "found at" << rule.info(); + return r; + } + return new ForwardingRepository(rule.name, r, rule.prefix); +} + static QString marksFileName(QString name) { name.replace('/', '_'); name.prepend("marks-"); return name; } -Repository::Repository(const Rules::Repository &rule) +FastImportRepository::FastImportRepository(const Rules::Repository &rule) : name(rule.name), prefix(rule.forwardTo), fastImport(name), commitCount(0), outstandingTransactions(0), last_commit_mark(0), next_file_mark(maxMark), processHasStarted(false) { foreach (Rules::Repository::Branch branchRule, rule.branches) { Branch branch; branch.created = 1; branches.insert(branchRule.name, branch); } // create the default branch branches["master"].created = 1; fastImport.setWorkingDirectory(name); - if (!CommandLineParser::instance()->contains("dry-run")) { + if (!CommandLineParser::instance()->contains("dry-run") && !CommandLineParser::instance()->contains("create-dump")) { if (!QDir(name).exists()) { // repo doesn't exist yet. qDebug() << "Creating new repository" << name; QDir::current().mkpath(name); QProcess init; init.setWorkingDirectory(name); init.start("git", QStringList() << "--bare" << "init"); init.waitForFinished(-1); // Write description if (!rule.description.isEmpty()) { QFile fDesc(QDir(name).filePath("description")); if (fDesc.open(QIODevice::WriteOnly | QIODevice::Truncate | QIODevice::Text)) { fDesc.write(rule.description.toUtf8()); fDesc.putChar('\n'); fDesc.close(); } } { QFile marks(name + "/" + marksFileName(name)); marks.open(QIODevice::WriteOnly); marks.close(); } } } } static QString logFileName(QString name) { name.replace('/', '_'); - name.prepend("log-"); + if (CommandLineParser::instance()->contains("create-dump")) + name.append(".fi"); + else + name.prepend("log-"); return name; } -static int lastValidMark(QString name) +static mark_t lastValidMark(QString name) { QFile marksfile(name + "/" + marksFileName(name)); if (!marksfile.open(QIODevice::ReadOnly)) return 0; - int prev_mark = 0; + qDebug() << "marksfile " << marksfile.fileName() ; + mark_t prev_mark = 0; int lineno = 0; while (!marksfile.atEnd()) { QString line = marksfile.readLine(); ++lineno; if (line.isEmpty()) continue; - int mark = 0; + mark_t mark = 0; if (line[0] == ':') { int sp = line.indexOf(' '); if (sp != -1) { QString m = line.mid(1, sp-1); - mark = m.toInt(); + mark = m.toULongLong(); } } if (!mark) { - qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?"; + qCritical() << marksfile.fileName() << "line" << lineno << "marks file corrupt?" << "mark " << mark; return 0; } if (mark == prev_mark) { qCritical() << marksfile.fileName() << "line" << lineno << "marks file has duplicates"; return 0; } if (mark < prev_mark) { qCritical() << marksfile.fileName() << "line" << lineno << "marks file not sorted"; return 0; } if (mark > prev_mark + 1) break; prev_mark = mark; } return prev_mark; } -int Repository::setupIncremental(int &cutoff) +int FastImportRepository::setupIncremental(int &cutoff) { QFile logfile(logFileName(name)); if (!logfile.exists()) return 1; logfile.open(QIODevice::ReadWrite); QRegExp progress("progress SVN r(\\d+) branch (.*) = :(\\d+)"); - int last_valid_mark = lastValidMark(name); + mark_t last_valid_mark = lastValidMark(name); int last_revnum = 0; qint64 pos = 0; int retval = 0; QString bkup = logfile.fileName() + ".old"; while (!logfile.atEnd()) { pos = logfile.pos(); QByteArray line = logfile.readLine(); int hash = line.indexOf('#'); if (hash != -1) line.truncate(hash); line = line.trimmed(); if (line.isEmpty()) continue; if (!progress.exactMatch(line)) continue; int revnum = progress.cap(1).toInt(); QString branch = progress.cap(2); - int mark = progress.cap(3).toInt(); + mark_t mark = progress.cap(3).toULongLong(); if (revnum >= cutoff) goto beyond_cutoff; if (revnum < last_revnum) qWarning() << "WARN:" << name << "revision numbers are not monotonic: " << "got" << QString::number(last_revnum) << "and then" << QString::number(revnum); if (mark > last_valid_mark) { qWarning() << "WARN:" << name << "unknown commit mark found: rewinding -- did you hit Ctrl-C?"; cutoff = revnum; goto beyond_cutoff; } last_revnum = revnum; if (last_commit_mark < mark) last_commit_mark = mark; Branch &br = branches[branch]; if (!br.created || !mark || br.marks.isEmpty() || !br.marks.last()) br.created = revnum; br.commits.append(revnum); br.marks.append(mark); } retval = last_revnum + 1; if (retval == cutoff) /* * If a stale backup file exists already, remove it, so that * we don't confuse ourselves in 'restoreLog()' */ QFile::remove(bkup); return retval; beyond_cutoff: // backup file, since we'll truncate QFile::remove(bkup); logfile.copy(bkup); // truncate, so that we ignore the rest of the revisions qDebug() << name << "truncating history to revision" << cutoff; logfile.resize(pos); return cutoff; } -void Repository::restoreLog() +void FastImportRepository::restoreLog() { QString file = logFileName(name); QString bkup = file + ".old"; if (!QFile::exists(bkup)) return; QFile::remove(file); QFile::rename(bkup, file); } -Repository::~Repository() +FastImportRepository::~FastImportRepository() { Q_ASSERT(outstandingTransactions == 0); closeFastImport(); } -void Repository::closeFastImport() +void FastImportRepository::closeFastImport() { if (fastImport.state() != QProcess::NotRunning) { fastImport.write("checkpoint\n"); fastImport.waitForBytesWritten(-1); fastImport.closeWriteChannel(); if (!fastImport.waitForFinished()) { fastImport.terminate(); if (!fastImport.waitForFinished(200)) qWarning() << "WARN: git-fast-import for repository" << name << "did not die"; } } processHasStarted = false; processCache.remove(this); } -void Repository::reloadBranches() +void FastImportRepository::reloadBranches() { bool reset_notes = false; foreach (QString branch, branches.keys()) { Branch &br = branches[branch]; if (br.marks.isEmpty() || !br.marks.last()) continue; reset_notes = true; QByteArray branchRef = branch.toUtf8(); if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); fastImport.write("reset " + branchRef + "\nfrom :" + QByteArray::number(br.marks.last()) + "\n\n" "progress Branch " + branchRef + " reloaded\n"); } if (reset_notes && CommandLineParser::instance()->contains("add-metadata-notes")) { fastImport.write("reset refs/notes/commits\nfrom :" + QByteArray::number(maxMark + 1) + "\n"); } } -int Repository::markFrom(const QString &branchFrom, int branchRevNum, QByteArray &branchFromDesc) +long long FastImportRepository::markFrom(const QString &branchFrom, int branchRevNum, QByteArray &branchFromDesc) { Branch &brFrom = branches[branchFrom]; if (!brFrom.created) return -1; if (brFrom.commits.isEmpty()) { return -1; } if (branchRevNum == brFrom.commits.last()) { return brFrom.marks.last(); } QVector::const_iterator it = qUpperBound(brFrom.commits, branchRevNum); if (it == brFrom.commits.begin()) { return 0; } int closestCommit = *--it; if (!branchFromDesc.isEmpty()) { branchFromDesc += " at r" + QByteArray::number(branchRevNum); if (closestCommit != branchRevNum) { branchFromDesc += " => r" + QByteArray::number(closestCommit); } } return brFrom.marks[it - brFrom.commits.begin()]; } -int Repository::createBranch(const QString &branch, int revnum, - const QString &branchFrom, int branchRevNum) +int FastImportRepository::createBranch(const QString &branch, int revnum, + const QString &branchFrom, int branchRevNum) { QByteArray branchFromDesc = "from branch " + branchFrom.toUtf8(); - int mark = markFrom(branchFrom, branchRevNum, branchFromDesc); + long long mark = markFrom(branchFrom, branchRevNum, branchFromDesc); if (mark == -1) { qCritical() << branch << "in repository" << name << "is branching from branch" << branchFrom << "but the latter doesn't exist. Can't continue."; return EXIT_FAILURE; } QByteArray branchFromRef = ":" + QByteArray::number(mark); if (!mark) { qWarning() << "WARN:" << branch << "in repository" << name << "is branching but no exported commits exist in repository" << "creating an empty branch."; branchFromRef = branchFrom.toUtf8(); if (!branchFromRef.startsWith("refs/")) branchFromRef.prepend("refs/heads/"); branchFromDesc += ", deleted/unknown"; } qDebug() << "Creating branch:" << branch << "from" << branchFrom << "(" << branchRevNum << branchFromDesc << ")"; // Preserve note branches[branch].note = branches.value(branchFrom).note; return resetBranch(branch, revnum, mark, branchFromRef, branchFromDesc); } -int Repository::deleteBranch(const QString &branch, int revnum) +int FastImportRepository::deleteBranch(const QString &branch, int revnum) { static QByteArray null_sha(40, '0'); return resetBranch(branch, revnum, 0, null_sha, "delete"); } -int Repository::resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment) +int FastImportRepository::resetBranch(const QString &branch, int revnum, mark_t mark, const QByteArray &resetTo, const QByteArray &comment) { QByteArray branchRef = branch.toUtf8(); if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); Branch &br = branches[branch]; QByteArray backupCmd; if (br.created && br.created != revnum && !br.marks.isEmpty() && br.marks.last()) { QByteArray backupBranch; if ((comment == "delete") && branchRef.startsWith("refs/heads/")) backupBranch = "refs/tags/backups/" + branchRef.mid(11) + "@" + QByteArray::number(revnum); else backupBranch = "refs/backups/r" + QByteArray::number(revnum) + branchRef.mid(4); qWarning() << "WARN: backing up branch" << branch << "to" << backupBranch; backupCmd = "reset " + backupBranch + "\nfrom " + branchRef + "\n\n"; } br.created = revnum; br.commits.append(revnum); br.marks.append(mark); QByteArray cmd = "reset " + branchRef + "\nfrom " + resetTo + "\n\n" "progress SVN r" + QByteArray::number(revnum) + " branch " + branch.toUtf8() + " = :" + QByteArray::number(mark) + " # " + comment + "\n\n"; if(comment == "delete") deletedBranches.append(backupCmd).append(cmd); else resetBranches.append(backupCmd).append(cmd); return EXIT_SUCCESS; } -void Repository::commit() +void FastImportRepository::commit() { if (deletedBranches.isEmpty() && resetBranches.isEmpty()) { return; } startFastImport(); fastImport.write(deletedBranches); fastImport.write(resetBranches); deletedBranches.clear(); resetBranches.clear(); } -Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix, - int revnum) +Repository::Transaction *FastImportRepository::newTransaction(const QString &branch, const QString &svnprefix, + int revnum) { if (!branches.contains(branch)) { qWarning() << "WARN: Transaction:" << branch << "is not a known branch in repository" << name << endl << "Going to create it automatically"; } Transaction *txn = new Transaction; txn->repository = this; txn->branch = branch.toUtf8(); txn->svnprefix = svnprefix.toUtf8(); txn->datetime = 0; txn->revnum = revnum; if ((++commitCount % CommandLineParser::instance()->optionArgument(QLatin1String("commit-interval"), QLatin1String("10000")).toInt()) == 0) { startFastImport(); // write everything to disk every 10000 commits fastImport.write("checkpoint\n"); qDebug() << "checkpoint!, marks file trunkated"; } outstandingTransactions++; return txn; } -void Repository::forgetTransaction(Transaction *) +void FastImportRepository::forgetTransaction(Transaction *) { if (!--outstandingTransactions) next_file_mark = maxMark; } -void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix, - int revnum, - const QByteArray &author, uint dt, - const QByteArray &log) +void FastImportRepository::createAnnotatedTag(const QString &ref, const QString &svnprefix, + int revnum, + const QByteArray &author, uint dt, + const QByteArray &log) { QString tagName = ref; if (tagName.startsWith("refs/tags/")) tagName.remove(0, 10); if (!annotatedTags.contains(tagName)) printf("Creating annotated tag %s (%s)\n", qPrintable(tagName), qPrintable(ref)); else printf("Re-creating annotated tag %s\n", qPrintable(tagName)); AnnotatedTag &tag = annotatedTags[tagName]; tag.supportingRef = ref; tag.svnprefix = svnprefix.toUtf8(); tag.revnum = revnum; tag.author = author; tag.log = log; tag.dt = dt; } -void Repository::finalizeTags() +void FastImportRepository::finalizeTags() { if (annotatedTags.isEmpty()) return; printf("Finalising tags for %s...", qPrintable(name)); startFastImport(); QHash::ConstIterator it = annotatedTags.constBegin(); for ( ; it != annotatedTags.constEnd(); ++it) { const QString &tagName = it.key(); const AnnotatedTag &tag = it.value(); QByteArray message = tag.log; if (!message.endsWith('\n')) message += '\n'; if (CommandLineParser::instance()->contains("add-metadata")) message += "\n" + formatMetadataMessage(tag.svnprefix, tag.revnum, tagName.toUtf8()); { QByteArray branchRef = tag.supportingRef.toUtf8(); if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); QByteArray s = "progress Creating annotated tag " + tagName.toUtf8() + " from ref " + branchRef + "\n" + "tag " + tagName.toUtf8() + "\n" + "from " + branchRef + "\n" + "tagger " + tag.author + ' ' + QByteArray::number(tag.dt) + " +0000" + "\n" + "data " + QByteArray::number( message.length() ) + "\n"; fastImport.write(s); } fastImport.write(message); fastImport.putChar('\n'); if (!fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString())); // Append note to the tip commit of the supporting ref. There is no // easy way to attach a note to the tag itself with fast-import. if (CommandLineParser::instance()->contains("add-metadata-notes")) { Repository::Transaction *txn = newTransaction(tag.supportingRef, tag.svnprefix, tag.revnum); txn->setAuthor(tag.author); txn->setDateTime(tag.dt); txn->commitNote(formatMetadataMessage(tag.svnprefix, tag.revnum, tagName.toUtf8()), true); delete txn; if (!fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString())); } printf(" %s", qPrintable(tagName)); fflush(stdout); } while (fastImport.bytesToWrite()) if (!fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString())); printf("\n"); } -void Repository::startFastImport() + +QByteArray +FastImportRepository::msgFilter(QByteArray msg) { + QByteArray output = msg; + + if (CommandLineParser::instance()->contains("msg-filter")) { + if (filterMsg.state() == QProcess::Running) + qFatal("filter process already running?"); + + filterMsg.start(CommandLineParser::instance()->optionArgument("msg-filter")); + + if(!(filterMsg.waitForStarted(-1))) + qFatal("Failed to Start Filter %d %s", __LINE__, qPrintable(filterMsg.errorString())); + + filterMsg.write(msg); + filterMsg.closeWriteChannel(); + filterMsg.waitForFinished(); + output = filterMsg.readAllStandardOutput(); + } + return output; +} + +void FastImportRepository::startFastImport() +{ processCache.touch(this); if (fastImport.state() == QProcess::NotRunning) { if (processHasStarted) qFatal("git-fast-import has been started once and crashed?"); processHasStarted = true; // start the process QString marksFile = marksFileName(name); QStringList marksOptions; marksOptions << "--import-marks=" + marksFile; marksOptions << "--export-marks=" + marksFile; marksOptions << "--force"; fastImport.setStandardOutputFile(logFileName(name), QIODevice::Append); fastImport.setProcessChannelMode(QProcess::MergedChannels); - if (!CommandLineParser::instance()->contains("dry-run")) { + if (!CommandLineParser::instance()->contains("dry-run") && !CommandLineParser::instance()->contains("create-dump")) { fastImport.start("git", QStringList() << "fast-import" << marksOptions); } else { fastImport.start("/bin/cat", QStringList()); } fastImport.waitForStarted(-1); reloadBranches(); } } QByteArray Repository::formatMetadataMessage(const QByteArray &svnprefix, int revnum, const QByteArray &tag) { QByteArray msg = "svn path=" + svnprefix + "; revision=" + QByteArray::number(revnum); if (!tag.isEmpty()) msg += "; tag=" + tag; msg += "\n"; return msg; } -bool Repository::branchExists(const QString& branch) const +bool FastImportRepository::branchExists(const QString& branch) const { return branches.contains(branch); } -const QByteArray Repository::branchNote(const QString& branch) const +const QByteArray FastImportRepository::branchNote(const QString& branch) const { return branches.value(branch).note; } -void Repository::setBranchNote(const QString& branch, const QByteArray& noteText) +void FastImportRepository::setBranchNote(const QString& branch, const QByteArray& noteText) { if (branches.contains(branch)) branches[branch].note = noteText; } -Repository::Transaction::~Transaction() +bool FastImportRepository::hasPrefix() const { + return !prefix.isEmpty(); +} + +QString FastImportRepository::getName() const +{ + return name; +} + +Repository *FastImportRepository::getEffectiveRepository() +{ + return this; +} + +FastImportRepository::Transaction::~Transaction() +{ repository->forgetTransaction(this); } -void Repository::Transaction::setAuthor(const QByteArray &a) +void FastImportRepository::Transaction::setAuthor(const QByteArray &a) { author = a; } -void Repository::Transaction::setDateTime(uint dt) +void FastImportRepository::Transaction::setDateTime(uint dt) { datetime = dt; } -void Repository::Transaction::setLog(const QByteArray &l) +void FastImportRepository::Transaction::setLog(const QByteArray &l) { log = l; } -void Repository::Transaction::noteCopyFromBranch(const QString &branchFrom, int branchRevNum) +void FastImportRepository::Transaction::noteCopyFromBranch(const QString &branchFrom, int branchRevNum) { if(branch == branchFrom) { qWarning() << "WARN: Cannot merge inside a branch"; return; } static QByteArray dummy; - int mark = repository->markFrom(branchFrom, branchRevNum, dummy); + long long mark = repository->markFrom(branchFrom, branchRevNum, dummy); Q_ASSERT(dummy.isEmpty()); if (mark == -1) { qWarning() << "WARN:" << branch << "is copying from branch" << branchFrom << "but the latter doesn't exist. Continuing, assuming the files exist."; } else if (mark == 0) { qWarning() << "WARN: Unknown revision r" << QByteArray::number(branchRevNum) << ". Continuing, assuming the files exist."; } else { qWarning() << "WARN: repository " + repository->name + " branch " + branch + " has some files copied from " + branchFrom + "@" + QByteArray::number(branchRevNum); if (!merges.contains(mark)) { merges.append(mark); qDebug() << "adding" << branchFrom + "@" + QByteArray::number(branchRevNum) << ":" << mark << "as a merge point"; } else { qDebug() << "merge point already recorded"; } } } -void Repository::Transaction::deleteFile(const QString &path) +void FastImportRepository::Transaction::deleteFile(const QString &path) { QString pathNoSlash = repository->prefix + path; if(pathNoSlash.endsWith('/')) pathNoSlash.chop(1); deletedFiles.append(pathNoSlash); } -QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length) +QIODevice *FastImportRepository::Transaction::addFile(const QString &path, int mode, qint64 length) { - int mark = repository->next_file_mark--; + mark_t mark = repository->next_file_mark--; // in case the two mark allocations meet, we might as well just abort Q_ASSERT(mark > repository->last_commit_mark + 1); if (modifiedFiles.capacity() == 0) modifiedFiles.reserve(2048); modifiedFiles.append("M "); modifiedFiles.append(QByteArray::number(mode, 8)); modifiedFiles.append(" :"); modifiedFiles.append(QByteArray::number(mark)); modifiedFiles.append(' '); modifiedFiles.append(repository->prefix + path.toUtf8()); modifiedFiles.append("\n"); if (!CommandLineParser::instance()->contains("dry-run")) { repository->startFastImport(); repository->fastImport.writeNoLog("blob\nmark :"); repository->fastImport.writeNoLog(QByteArray::number(mark)); repository->fastImport.writeNoLog("\ndata "); repository->fastImport.writeNoLog(QByteArray::number(length)); repository->fastImport.writeNoLog("\n", 1); } return &repository->fastImport; } -void Repository::Transaction::commitNote(const QByteArray ¬eText, bool append, const QByteArray &commit) +void FastImportRepository::Transaction::commitNote(const QByteArray ¬eText, bool append, const QByteArray &commit) { QByteArray branchRef = branch; if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); const QByteArray &commitRef = commit.isNull() ? branchRef : commit; QByteArray message = "Adding Git note for current " + commitRef + "\n"; QByteArray text = noteText; if (append && commit.isNull() && repository->branchExists(branch) && !repository->branchNote(branch).isEmpty()) { text = repository->branchNote(branch) + text; message = "Appending Git note for current " + commitRef + "\n"; } QByteArray s(""); s.append("commit refs/notes/commits\n"); s.append("mark :" + QByteArray::number(maxMark + 1) + "\n"); s.append("committer " + author + " " + QString::number(datetime) + " +0000" + "\n"); s.append("data " + QString::number(message.length()) + "\n"); s.append(message + "\n"); s.append("N inline " + commitRef + "\n"); s.append("data " + QString::number(text.length()) + "\n"); s.append(text + "\n"); repository->fastImport.write(s); if (commit.isNull()) { repository->setBranchNote(QString::fromUtf8(branch), text); } } -void Repository::Transaction::commit() +void FastImportRepository::Transaction::commit() { repository->startFastImport(); // We might be tempted to use the SVN revision number as the fast-import commit mark. // However, a single SVN revision can modify multple branches, and thus lead to multiple // commits in the same repo. So, we need to maintain a separate commit mark counter. - int mark = ++repository->last_commit_mark; + mark_t mark = ++repository->last_commit_mark; // in case the two mark allocations meet, we might as well just abort Q_ASSERT(mark < repository->next_file_mark - 1); // create the commit message QByteArray message = log; if (!message.endsWith('\n')) message += '\n'; if (CommandLineParser::instance()->contains("add-metadata")) message += "\n" + Repository::formatMetadataMessage(svnprefix, revnum); - int parentmark = 0; + // Call external message filter if provided + message = repository->msgFilter(message); + + mark_t parentmark = 0; Branch &br = repository->branches[branch]; if (br.created && !br.marks.isEmpty() && br.marks.last()) { parentmark = br.marks.last(); } else { qWarning() << "WARN: Branch" << branch << "in repository" << repository->name << "doesn't exist at revision" << revnum << "-- did you resume from the wrong revision?"; br.created = revnum; } br.commits.append(revnum); br.marks.append(mark); QByteArray branchRef = branch; if (!branchRef.startsWith("refs/")) branchRef.prepend("refs/heads/"); QByteArray s(""); s.append("commit " + branchRef + "\n"); s.append("mark :" + QByteArray::number(mark) + "\n"); s.append("committer " + author + " " + QString::number(datetime).toUtf8() + " +0000" + "\n"); s.append("data " + QString::number(message.length()) + "\n"); s.append(message + "\n"); repository->fastImport.write(s); // note some of the inferred merges QByteArray desc = ""; - int i = !!parentmark; // if parentmark != 0, there's at least one parent + mark_t i = !!parentmark; // if parentmark != 0, there's at least one parent if(log.contains("This commit was manufactured by cvs2svn") && merges.count() > 1) { qSort(merges); repository->fastImport.write("merge :" + QByteArray::number(merges.last()) + "\n"); merges.pop_back(); qWarning() << "WARN: Discarding all but the highest merge point as a workaround for cvs2svn created branch/tag" << "Discarded marks:" << merges; } else { - foreach (const int merge, merges) { + foreach (const mark_t merge, merges) { if (merge == parentmark) { qDebug() << "Skipping marking" << merge << "as a merge point as it matches the parent"; continue; } if (++i > 16) { // FIXME: options: // (1) ignore the 16 parent limit // (2) don't emit more than 16 parents // (3) create another commit on branch to soak up additional parents // we've chosen option (2) for now, since only artificial commits // created by cvs2svn seem to have this issue qWarning() << "WARN: too many merge parents"; break; } QByteArray m = " :" + QByteArray::number(merge); desc += m; repository->fastImport.write("merge" + m + "\n"); } } // write the file deletions if (deletedFiles.contains("")) repository->fastImport.write("deleteall\n"); else foreach (QString df, deletedFiles) repository->fastImport.write("D " + df.toUtf8() + "\n"); // write the file modifications repository->fastImport.write(modifiedFiles); repository->fastImport.write("\nprogress SVN r" + QByteArray::number(revnum) + " branch " + branch + " = :" + QByteArray::number(mark) + (desc.isEmpty() ? "" : " # merge from") + desc + "\n\n"); printf(" %d modifications from SVN %s to %s/%s", deletedFiles.count() + modifiedFiles.count('\n'), svnprefix.data(), qPrintable(repository->name), branch.data()); // Commit metadata note if requested if (CommandLineParser::instance()->contains("add-metadata-notes")) commitNote(Repository::formatMetadataMessage(svnprefix, revnum), false); while (repository->fastImport.bytesToWrite()) if (!repository->fastImport.waitForBytesWritten(-1)) qFatal("Failed to write to process: %s for repository %s", qPrintable(repository->fastImport.errorString()), qPrintable(repository->name)); } Index: user/uqs/git_conv/svn2git/src/repository.h =================================================================== --- user/uqs/git_conv/svn2git/src/repository.h (revision 311975) +++ user/uqs/git_conv/svn2git/src/repository.h (revision 311976) @@ -1,205 +1,149 @@ /* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #ifndef REPOSITORY_H #define REPOSITORY_H #include #include #include #include #include "ruleparser.h" #include "CommandLineParser.h" class LoggingQProcess : public QProcess { QFile log; bool logging; public: LoggingQProcess(const QString filename) : QProcess(), log() { if(CommandLineParser::instance()->contains("debug-rules")) { logging = true; QString name = filename; name.replace('/', '_'); name.prepend("gitlog-"); log.setFileName(name); log.open(QIODevice::WriteOnly); } else { logging = false; } }; ~LoggingQProcess() { if(logging) { log.close(); } }; qint64 write(const char *data) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.write(data); } return QProcess::write(data); } qint64 write(const char *data, qint64 length) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.write(data); } return QProcess::write(data, length); } qint64 write(const QByteArray &data) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.write(data); } return QProcess::write(data); } qint64 writeNoLog(const char *data) { Q_ASSERT(state() == QProcess::Running); return QProcess::write(data); } qint64 writeNoLog(const char *data, qint64 length) { Q_ASSERT(state() == QProcess::Running); return QProcess::write(data, length); } qint64 writeNoLog(const QByteArray &data) { Q_ASSERT(state() == QProcess::Running); return QProcess::write(data); } bool putChar( char c) { Q_ASSERT(state() == QProcess::Running); if(logging) { log.putChar(c); } return QProcess::putChar(c); } }; - class Repository { public: class Transaction { Q_DISABLE_COPY(Transaction) - friend class Repository; - - Repository *repository; - QByteArray branch; - QByteArray svnprefix; - QByteArray author; - QByteArray log; - uint datetime; - int revnum; - - QVector merges; - - QStringList deletedFiles; - QByteArray modifiedFiles; - - inline Transaction() {} + protected: + Transaction() {} public: - ~Transaction(); - void commit(); + virtual ~Transaction() {} + virtual void commit() = 0; - void setAuthor(const QByteArray &author); - void setDateTime(uint dt); - void setLog(const QByteArray &log); + virtual void setAuthor(const QByteArray &author) = 0; + virtual void setDateTime(uint dt) = 0; + virtual void setLog(const QByteArray &log) = 0; - void noteCopyFromBranch (const QString &prevbranch, int revFrom); + virtual void noteCopyFromBranch (const QString &prevbranch, int revFrom) = 0; - void deleteFile(const QString &path); - QIODevice *addFile(const QString &path, int mode, qint64 length); + virtual void deleteFile(const QString &path) = 0; + virtual QIODevice *addFile(const QString &path, int mode, qint64 length) = 0; - void commitNote(const QByteArray ¬eText, bool append, - const QByteArray &commit = QByteArray()); + virtual void commitNote(const QByteArray ¬eText, bool append, + const QByteArray &commit = QByteArray()) = 0; }; - Repository(const Rules::Repository &rule); - int setupIncremental(int &cutoff); - void restoreLog(); - ~Repository(); + virtual int setupIncremental(int &cutoff) = 0; + virtual void restoreLog() = 0; + virtual ~Repository() {} - void reloadBranches(); - int createBranch(const QString &branch, int revnum, - const QString &branchFrom, int revFrom); - int deleteBranch(const QString &branch, int revnum); - Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum); + virtual void reloadBranches() = 0; + virtual int createBranch(const QString &branch, int revnum, + const QString &branchFrom, int revFrom) = 0; + virtual int deleteBranch(const QString &branch, int revnum) = 0; + virtual Repository::Transaction *newTransaction(const QString &branch, const QString &svnprefix, int revnum) = 0; - void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, - const QByteArray &author, uint dt, - const QByteArray &log); - void finalizeTags(); - void commit(); + virtual void createAnnotatedTag(const QString &name, const QString &svnprefix, int revnum, + const QByteArray &author, uint dt, + const QByteArray &log) = 0; + virtual void finalizeTags() = 0; + virtual void commit() = 0; static QByteArray formatMetadataMessage(const QByteArray &svnprefix, int revnum, const QByteArray &tag = QByteArray()); - bool branchExists(const QString& branch) const; - const QByteArray branchNote(const QString& branch) const; - void setBranchNote(const QString& branch, const QByteArray& noteText); + virtual bool branchExists(const QString& branch) const = 0; + virtual const QByteArray branchNote(const QString& branch) const = 0; + virtual void setBranchNote(const QString& branch, const QByteArray& noteText) = 0; -private: - struct Branch - { - int created; - QVector commits; - QVector marks; - QByteArray note; - }; - struct AnnotatedTag - { - QString supportingRef; - QByteArray svnprefix; - QByteArray author; - QByteArray log; - uint dt; - int revnum; - }; + virtual bool hasPrefix() const = 0; - QHash branches; - QHash annotatedTags; - QString name; - QString prefix; - LoggingQProcess fastImport; - int commitCount; - int outstandingTransactions; - QByteArray deletedBranches; - QByteArray resetBranches; + virtual QString getName() const = 0; + virtual Repository *getEffectiveRepository() = 0; +}; - /* starts at 0, and counts up. */ - int last_commit_mark; +Repository *createRepository(const Rules::Repository &rule, const QHash &repositories); - /* starts at maxMark and counts down. Reset after each SVN revision */ - int next_file_mark; - - bool processHasStarted; - - void startFastImport(); - void closeFastImport(); - - // called when a transaction is deleted - void forgetTransaction(Transaction *t); - - int resetBranch(const QString &branch, int revnum, int mark, const QByteArray &resetTo, const QByteArray &comment); - int markFrom(const QString &branchFrom, int branchRevNum, QByteArray &desc); - - friend class ProcessCache; - Q_DISABLE_COPY(Repository) -}; #endif Index: user/uqs/git_conv/svn2git/src/ruleparser.cpp =================================================================== --- user/uqs/git_conv/svn2git/src/ruleparser.cpp (revision 311975) +++ user/uqs/git_conv/svn2git/src/ruleparser.cpp (revision 311976) @@ -1,404 +1,414 @@ /* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include #include #include #include #include "ruleparser.h" #include "CommandLineParser.h" RulesList::RulesList(const QString &filenames) : m_filenames(filenames) { } RulesList::~RulesList() {} void RulesList::load() { foreach(const QString filename, m_filenames.split(',') ) { qDebug() << "Loading rules from:" << filename; Rules *rules = new Rules(filename); m_rules.append(rules); rules->load(); m_allrepositories.append(rules->repositories()); QList matchRules = rules->matchRules(); m_allMatchRules.append( QList(matchRules)); } } const QList RulesList::allRepositories() const { return m_allrepositories; } const QList > RulesList::allMatchRules() const { return m_allMatchRules; } const QList RulesList::rules() const { return m_rules; } Rules::Rules(const QString &fn) : filename(fn) { } Rules::~Rules() { } const QList Rules::repositories() const { return m_repositories; } const QList Rules::matchRules() const { return m_matchRules; } Rules::Match::Substitution Rules::parseSubstitution(const QString &string) { if (string.at(0) != 's' || string.length() < 5) return Match::Substitution(); const QChar sep = string.at(1); if (string.at(string.length() - 1) != sep) return Match::Substitution(); int i = 2, end = 0; Match::Substitution subst; // Separator might have been escaped with a backslash while (i > end) { int backslashCount = 0; if ((end = string.indexOf(sep, i)) > -1) { for (i = end - 1; i >= 2; i--) { if (string.at(i) == '\\') backslashCount++; else break; } } else { return Match::Substitution(); // error } if (backslashCount % 2 != 0) { // Separator was escaped. Search for another one i = end + 1; } } // Found the end of the pattern subst.pattern = QRegExp(string.mid(2, end - 2)); if (!subst.pattern.isValid()) return Match::Substitution(); // error subst.replacement = string.mid(end + 1, string.length() - 1 - end - 1); return subst; } void Rules::load() { load(filename); } void Rules::load(const QString &filename) { qDebug() << "Loading rules from" << filename; // initialize the regexps we will use QRegExp repoLine("create repository\\s+(\\S+)", Qt::CaseInsensitive); QString varRegex("[A-Za-z0-9_]+"); QRegExp matchLine("match\\s+(.*)", Qt::CaseInsensitive); QRegExp matchActionLine("action\\s+(\\w+)", Qt::CaseInsensitive); QRegExp matchRepoLine("repository\\s+(\\S+)", Qt::CaseInsensitive); QRegExp matchDescLine("description\\s+(.+)$", Qt::CaseInsensitive); QRegExp matchRepoSubstLine("substitute repository\\s+(.+)$", Qt::CaseInsensitive); QRegExp matchBranchLine("branch\\s+(\\S+)", Qt::CaseInsensitive); QRegExp matchBranchSubstLine("substitute branch\\s+(.+)$", Qt::CaseInsensitive); QRegExp matchRevLine("(min|max) revision (\\d+)", Qt::CaseInsensitive); QRegExp matchAnnotateLine("annotated\\s+(\\S+)", Qt::CaseInsensitive); QRegExp matchPrefixLine("prefix\\s+(.*)$", Qt::CaseInsensitive); QRegExp declareLine("declare\\s+("+varRegex+")\\s*=\\s*(\\S+)", Qt::CaseInsensitive); QRegExp variableLine("\\$\\{("+varRegex+")(\\|[^}$]*)?\\}", Qt::CaseInsensitive); QRegExp includeLine("include\\s+(.*)", Qt::CaseInsensitive); enum { ReadingNone, ReadingRepository, ReadingMatch } state = ReadingNone; Repository repo; Match match; int lineNumber = 0; QFile file(filename); if (!file.open(QIODevice::ReadOnly)) qFatal("Could not read the rules file: %s", qPrintable(filename)); QTextStream s(&file); QStringList lines = s.readAll().split('\n', QString::KeepEmptyParts); QStringList::iterator it; for(it = lines.begin(); it != lines.end(); ++it) { ++lineNumber; QString origLine = *it; QString line = origLine; int hash = line.indexOf('#'); if (hash != -1) line.truncate(hash); line = line.trimmed(); if (line.isEmpty()) continue; bool isIncludeRule = includeLine.exactMatch(line); if (isIncludeRule) { int index = filename.lastIndexOf("/"); QString includeFile = filename.left( index + 1) + includeLine.cap(1); load(includeFile); } else { while( variableLine.indexIn(line) != -1 ) { QString replacement; if (m_variables.contains(variableLine.cap(1))) { replacement = m_variables[variableLine.cap(1)]; } else { if (variableLine.cap(2).startsWith('|')) { replacement = variableLine.cap(2).mid(1); } else { qFatal("Undeclared variable: %s", qPrintable(variableLine.cap(1))); } } line = line.replace(variableLine.cap(0), replacement); } if (state == ReadingRepository) { if (matchBranchLine.exactMatch(line)) { Repository::Branch branch; branch.name = matchBranchLine.cap(1); repo.branches += branch; continue; } else if (matchDescLine.exactMatch(line)) { repo.description = matchDescLine.cap(1); continue; } else if (matchRepoLine.exactMatch(line)) { repo.forwardTo = matchRepoLine.cap(1); continue; } else if (matchPrefixLine.exactMatch(line)) { repo.prefix = matchPrefixLine.cap(1); continue; } else if (line == "end repository") { + if (!repo.forwardTo.isEmpty() + && !repo.description.isEmpty()) { + + qFatal("Specifing repository and description on repository is invalid on line %d", lineNumber); + } + + if (!repo.forwardTo.isEmpty() + && !repo.branches.isEmpty()) { + + qFatal("Specifing repository and branches on repository is invalid on line %d", lineNumber); + } + m_repositories += repo; { // clear out 'repo' Repository temp; std::swap(repo, temp); } state = ReadingNone; continue; } } else if (state == ReadingMatch) { if (matchRepoLine.exactMatch(line)) { match.repository = matchRepoLine.cap(1); continue; } else if (matchBranchLine.exactMatch(line)) { match.branch = matchBranchLine.cap(1); continue; } else if (matchRepoSubstLine.exactMatch(line)) { Match::Substitution subst = parseSubstitution(matchRepoSubstLine.cap(1)); if (!subst.isValid()) { qFatal("Malformed substitution in rules file: line %d: %s", lineNumber, qPrintable(origLine)); } match.repo_substs += subst; continue; } else if (matchBranchSubstLine.exactMatch(line)) { Match::Substitution subst = parseSubstitution(matchBranchSubstLine.cap(1)); if (!subst.isValid()) { qFatal("Malformed substitution in rules file: line %d: %s", lineNumber, qPrintable(origLine)); } match.branch_substs += subst; continue; } else if (matchRevLine.exactMatch(line)) { if (matchRevLine.cap(1) == "min") match.minRevision = matchRevLine.cap(2).toInt(); else // must be max match.maxRevision = matchRevLine.cap(2).toInt(); continue; } else if (matchPrefixLine.exactMatch(line)) { match.prefix = matchPrefixLine.cap(1); if( match.prefix.startsWith('/')) match.prefix = match.prefix.mid(1); continue; } else if (matchActionLine.exactMatch(line)) { QString action = matchActionLine.cap(1); if (action == "export") match.action = Match::Export; else if (action == "ignore") match.action = Match::Ignore; else if (action == "recurse") match.action = Match::Recurse; else qFatal("Invalid action \"%s\" on line %d", qPrintable(action), lineNumber); continue; } else if (matchAnnotateLine.exactMatch(line)) { match.annotate = matchAnnotateLine.cap(1) == "true"; continue; } else if (line == "end match") { if (!match.repository.isEmpty()) match.action = Match::Export; m_matchRules += match; Stats::instance()->addRule(match); state = ReadingNone; continue; } } bool isRepositoryRule = repoLine.exactMatch(line); bool isMatchRule = matchLine.exactMatch(line); bool isVariableRule = declareLine.exactMatch(line); if (isRepositoryRule) { // repository rule state = ReadingRepository; repo = Repository(); // clear repo.name = repoLine.cap(1); repo.lineNumber = lineNumber; repo.filename = filename; } else if (isMatchRule) { // match rule state = ReadingMatch; match = Match(); match.rx = QRegExp(matchLine.cap(1), Qt::CaseSensitive, QRegExp::RegExp2); if( !match.rx.isValid() ) qFatal("Malformed regular expression '%s' in file:'%s':%d, Error: %s", qPrintable(matchLine.cap(1)), qPrintable(filename), lineNumber, qPrintable(match.rx.errorString())); match.lineNumber = lineNumber; match.filename = filename; } else if (isVariableRule) { QString variable = declareLine.cap(1); QString value = declareLine.cap(2); m_variables.insert(variable, value); } else { qFatal("Malformed line in rules file: line %d: %s", lineNumber, qPrintable(origLine)); } } } } Stats *Stats::self = 0; class Stats::Private { public: Private(); void printStats() const; void ruleMatched(const Rules::Match &rule, const int rev); void addRule(const Rules::Match &rule); private: - QMap m_usedRules; + QMap m_usedRules; }; Stats::Stats() : d(new Private()) { use = CommandLineParser::instance()->contains("stats"); } Stats::~Stats() { delete d; } void Stats::init() { if(self) delete self; self = new Stats(); } Stats* Stats::instance() { return self; } void Stats::printStats() const { if(use) d->printStats(); } void Stats::ruleMatched(const Rules::Match &rule, const int rev) { if(use) d->ruleMatched(rule, rev); } void Stats::addRule( const Rules::Match &rule) { if(use) d->addRule(rule); } Stats::Private::Private() { } void Stats::Private::printStats() const { printf("\nRule stats\n"); - foreach(const QString name, m_usedRules.keys()) { - printf("%s was matched %i times\n", qPrintable(name), m_usedRules[name]); + foreach(const Rules::Match rule, m_usedRules.keys()) { + printf("%s was matched %i times\n", qPrintable(rule.info()), m_usedRules[rule]); } } void Stats::Private::ruleMatched(const Rules::Match &rule, const int rev) { Q_UNUSED(rev); - const QString name = rule.info(); - if(!m_usedRules.contains(name)) { - m_usedRules.insert(name, 1); - qWarning() << "WARN: New match rule, should have been added when created."; + if(!m_usedRules.contains(rule)) { + m_usedRules.insert(rule, 1); + qWarning() << "WARN: New match rule" << rule.info() << ", should have been added when created."; } else { - m_usedRules[name]++; + m_usedRules[rule]++; } } void Stats::Private::addRule( const Rules::Match &rule) { - const QString name = rule.info(); - if(m_usedRules.contains(name)) - qWarning() << "WARN: Rule" << name << "was added multiple times."; - m_usedRules.insert(name, 0); + if(m_usedRules.contains(rule)) + qWarning() << "WARN: Rule" << rule.info() << "was added multiple times."; + m_usedRules.insert(rule, 0); } #ifndef QT_NO_DEBUG_STREAM QDebug operator<<(QDebug s, const Rules::Match &rule) { s.nospace() << rule.info(); return s.space(); } #endif Index: user/uqs/git_conv/svn2git/src/ruleparser.h =================================================================== --- user/uqs/git_conv/svn2git/src/ruleparser.h (revision 311975) +++ user/uqs/git_conv/svn2git/src/ruleparser.h (revision 311976) @@ -1,149 +1,154 @@ /* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #ifndef RULEPARSER_H #define RULEPARSER_H #include #include #include #include #include #include class Rules { public: struct Rule { QString filename; int lineNumber; Rule() : lineNumber(0) {} }; struct Repository : Rule { struct Branch { QString name; }; QString name; QList branches; QString description; QString forwardTo; QString prefix; Repository() { } const QString info() const { const QString info = Rule::filename % ":" % QByteArray::number(Rule::lineNumber); return info; } }; struct Match : Rule { struct Substitution { QRegExp pattern; QString replacement; bool isValid() { return !pattern.isEmpty(); } QString& apply(QString &string) { return string.replace(pattern, replacement); } }; QRegExp rx; QString repository; QList repo_substs; QString branch; QList branch_substs; QString prefix; int minRevision; int maxRevision; bool annotate; enum Action { Ignore, Export, Recurse } action; Match() : minRevision(-1), maxRevision(-1), annotate(false), action(Ignore) { } + bool operator<(const Match other) const { + if (filename != other.filename) + return filename < other.filename; + return lineNumber < other.lineNumber; + } const QString info() const { const QString info = Rule::filename % ":" % QByteArray::number(Rule::lineNumber) % " " % rx.pattern(); return info; } }; Rules(const QString &filename); ~Rules(); const QList repositories() const; const QList matchRules() const; Match::Substitution parseSubstitution(const QString &string); void load(); private: void load(const QString &filename); QString filename; QList m_repositories; QList m_matchRules; QMap m_variables; }; class RulesList { public: RulesList( const QString &filenames); ~RulesList(); const QList allRepositories() const; const QList > allMatchRules() const; const QList rules() const; void load(); private: QString m_filenames; QList m_rules; QList m_allrepositories; QList > m_allMatchRules; }; class Stats { public: static Stats *instance(); void printStats() const; void ruleMatched(const Rules::Match &rule, const int rev = -1); void addRule( const Rules::Match &rule); static void init(); ~Stats(); private: Stats(); class Private; Private * const d; static Stats *self; bool use; }; #ifndef QT_NO_DEBUG_STREAM class QDebug; QDebug operator<<(QDebug, const Rules::Match &); #endif #endif Index: user/uqs/git_conv/svn2git/src/svn.cpp =================================================================== --- user/uqs/git_conv/svn2git/src/svn.cpp (revision 311975) +++ user/uqs/git_conv/svn2git/src/svn.cpp (revision 311976) @@ -1,922 +1,1052 @@ /* * Copyright (C) 2007 Thiago Macieira * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ /* * Based on svn-fast-export by Chris Lee * License: MIT * URL: git://repo.or.cz/fast-import.git http://repo.or.cz/w/fast-export.git */ #define _XOPEN_SOURCE #define _LARGEFILE_SUPPORT #define _LARGEFILE64_SUPPORT #include "svn.h" #include "CommandLineParser.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "repository.h" #undef SVN_ERR #define SVN_ERR(expr) SVN_INT_ERR(expr) typedef QList MatchRuleList; typedef QHash RepositoryHash; typedef QHash IdentityHash; class AprAutoPool { apr_pool_t *pool; AprAutoPool(const AprAutoPool &); AprAutoPool &operator=(const AprAutoPool &); public: inline AprAutoPool(apr_pool_t *parent = NULL) - { pool = svn_pool_create(parent); } - inline ~AprAutoPool() - { svn_pool_destroy(pool); } + { + pool = svn_pool_create(parent); + } + inline ~AprAutoPool() + { + svn_pool_destroy(pool); + } inline void clear() { svn_pool_clear(pool); } inline apr_pool_t *data() const { return pool; } inline operator apr_pool_t *() const { return pool; } }; class SvnPrivate { public: QList allMatchRules; RepositoryHash repositories; IdentityHash identities; QString userdomain; SvnPrivate(const QString &pathToRepository); ~SvnPrivate(); int youngestRevision(); int exportRevision(int revnum); int openRepository(const QString &pathToRepository); private: AprAutoPool global_pool; + AprAutoPool scratch_pool; svn_fs_t *fs; svn_revnum_t youngest_rev; }; void Svn::initialize() { // initialize APR or exit if (apr_initialize() != APR_SUCCESS) { fprintf(stderr, "You lose at apr_initialize().\n"); exit(1); } // static destructor static struct Destructor { ~Destructor() { apr_terminate(); } } destructor; } Svn::Svn(const QString &pathToRepository) : d(new SvnPrivate(pathToRepository)) { } Svn::~Svn() { delete d; } void Svn::setMatchRules(const QList &allMatchRules) { d->allMatchRules = allMatchRules; } void Svn::setRepositories(const RepositoryHash &repositories) { d->repositories = repositories; } void Svn::setIdentityMap(const IdentityHash &identityMap) { d->identities = identityMap; } void Svn::setIdentityDomain(const QString &identityDomain) { d->userdomain = identityDomain; } int Svn::youngestRevision() { return d->youngestRevision(); } bool Svn::exportRevision(int revnum) { return d->exportRevision(revnum) == EXIT_SUCCESS; } SvnPrivate::SvnPrivate(const QString &pathToRepository) - : global_pool(NULL) + : global_pool(NULL) , scratch_pool(NULL) { if( openRepository(pathToRepository) != EXIT_SUCCESS) { qCritical() << "Failed to open repository"; exit(1); } // get the youngest revision svn_fs_youngest_rev(&youngest_rev, fs, global_pool); } -SvnPrivate::~SvnPrivate() -{ - svn_pool_destroy(global_pool); -} +SvnPrivate::~SvnPrivate() {} int SvnPrivate::youngestRevision() { return youngest_rev; } int SvnPrivate::openRepository(const QString &pathToRepository) { svn_repos_t *repos; QString path = pathToRepository; while (path.endsWith('/')) // no trailing slash allowed path = path.mid(0, path.length()-1); - SVN_ERR(svn_repos_open(&repos, QFile::encodeName(path), global_pool)); + SVN_ERR(svn_repos_open3(&repos, QFile::encodeName(path), NULL, global_pool, scratch_pool)); fs = svn_repos_fs(repos); return EXIT_SUCCESS; } enum RuleType { AnyRule = 0, NoIgnoreRule = 0x01, NoRecurseRule = 0x02 }; static MatchRuleList::ConstIterator findMatchRule(const MatchRuleList &matchRules, int revnum, const QString ¤t, int ruleMask = AnyRule) { MatchRuleList::ConstIterator it = matchRules.constBegin(), end = matchRules.constEnd(); for ( ; it != end; ++it) { if (it->minRevision > revnum) continue; if (it->maxRevision != -1 && it->maxRevision < revnum) continue; if (it->action == Rules::Match::Ignore && ruleMask & NoIgnoreRule) continue; if (it->action == Rules::Match::Recurse && ruleMask & NoRecurseRule) continue; if (it->rx.indexIn(current) == 0) { Stats::instance()->ruleMatched(*it, revnum); return it; } } // no match return end; } -static void splitPathName(const Rules::Match &rule, const QString &pathName, QString *svnprefix_p, - QString *repository_p, QString *branch_p, QString *path_p) -{ - QString svnprefix = pathName; - svnprefix.truncate(rule.rx.matchedLength()); - - if (svnprefix_p) { - *svnprefix_p = svnprefix; - } - - if (repository_p) { - *repository_p = svnprefix; - repository_p->replace(rule.rx, rule.repository); - foreach (Rules::Match::Substitution subst, rule.repo_substs) { - subst.apply(*repository_p); - } - } - - if (branch_p) { - *branch_p = svnprefix; - branch_p->replace(rule.rx, rule.branch); - foreach (Rules::Match::Substitution subst, rule.branch_substs) { - subst.apply(*branch_p); - } - } - - if (path_p) { - QString prefix = svnprefix; - prefix.replace(rule.rx, rule.prefix); - *path_p = prefix + pathName.mid(svnprefix.length()); - } -} - static int pathMode(svn_fs_root_t *fs_root, const char *pathname, apr_pool_t *pool) { svn_string_t *propvalue; SVN_ERR(svn_fs_node_prop(&propvalue, fs_root, pathname, "svn:executable", pool)); int mode = 0100644; if (propvalue) mode = 0100755; return mode; } svn_error_t *QIODevice_write(void *baton, const char *data, apr_size_t *len) { QIODevice *device = reinterpret_cast(baton); device->write(data, *len); while (device->bytesToWrite() > 32*1024) { if (!device->waitForBytesWritten(-1)) { qFatal("Failed to write to process: %s", qPrintable(device->errorString())); return svn_error_createf(APR_EOF, SVN_NO_ERROR, "Failed to write to process: %s", qPrintable(device->errorString())); } } return SVN_NO_ERROR; } static svn_stream_t *streamForDevice(QIODevice *device, apr_pool_t *pool) { svn_stream_t *stream = svn_stream_create(device, pool); svn_stream_set_write(stream, QIODevice_write); return stream; } static int dumpBlob(Repository::Transaction *txn, svn_fs_root_t *fs_root, const char *pathname, const QString &finalPathName, apr_pool_t *pool) { AprAutoPool dumppool(pool); // what type is it? int mode = pathMode(fs_root, pathname, dumppool); svn_filesize_t stream_length; SVN_ERR(svn_fs_file_length(&stream_length, fs_root, pathname, dumppool)); svn_stream_t *in_stream, *out_stream; if (!CommandLineParser::instance()->contains("dry-run")) { // open the file SVN_ERR(svn_fs_file_contents(&in_stream, fs_root, pathname, dumppool)); } // maybe it's a symlink? svn_string_t *propvalue; SVN_ERR(svn_fs_node_prop(&propvalue, fs_root, pathname, "svn:special", dumppool)); if (propvalue) { apr_size_t len = strlen("link "); if (!CommandLineParser::instance()->contains("dry-run")) { QByteArray buf; buf.reserve(len); - SVN_ERR(svn_stream_read(in_stream, buf.data(), &len)); + SVN_ERR(svn_stream_read_full(in_stream, buf.data(), &len)); if (len == strlen("link ") && strncmp(buf, "link ", len) == 0) { mode = 0120000; stream_length -= len; } else { //this can happen if a link changed into a file in one commit qWarning("file %s is svn:special but not a symlink", pathname); // re-open the file as we tried to read "link " svn_stream_close(in_stream); SVN_ERR(svn_fs_file_contents(&in_stream, fs_root, pathname, dumppool)); } } } QIODevice *io = txn->addFile(finalPathName, mode, stream_length); if (!CommandLineParser::instance()->contains("dry-run")) { // open a generic svn_stream_t for the QIODevice out_stream = streamForDevice(io, dumppool); - SVN_ERR(svn_stream_copy(in_stream, out_stream, dumppool)); - svn_stream_close(out_stream); - svn_stream_close(in_stream); + SVN_ERR(svn_stream_copy3(in_stream, out_stream, NULL, NULL, dumppool)); // print an ending newline io->putChar('\n'); } return EXIT_SUCCESS; } static int recursiveDumpDir(Repository::Transaction *txn, svn_fs_root_t *fs_root, const QByteArray &pathname, const QString &finalPathName, apr_pool_t *pool) { // get the dir listing apr_hash_t *entries; SVN_ERR(svn_fs_dir_entries(&entries, fs_root, pathname, pool)); AprAutoPool dirpool(pool); // While we get a hash, put it in a map for sorted lookup, so we can // repeat the conversions and get the same git commit hashes. QMap map; for (apr_hash_index_t *i = apr_hash_first(pool, entries); i; i = apr_hash_next(i)) { const void *vkey; void *value; apr_hash_this(i, &vkey, NULL, &value); svn_fs_dirent_t *dirent = reinterpret_cast(value); map.insertMulti(QByteArray(dirent->name), dirent->kind); } QMapIterator i(map); while (i.hasNext()) { dirpool.clear(); i.next(); QByteArray entryName = pathname + '/' + i.key(); QString entryFinalName = finalPathName + QString::fromUtf8(i.key()); if (i.value() == svn_node_dir) { entryFinalName += '/'; if (recursiveDumpDir(txn, fs_root, entryName, entryFinalName, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } else if (i.value() == svn_node_file) { printf("+"); fflush(stdout); if (dumpBlob(txn, fs_root, entryName, entryFinalName, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } } return EXIT_SUCCESS; } static bool wasDir(svn_fs_t *fs, int revnum, const char *pathname, apr_pool_t *pool) { AprAutoPool subpool(pool); svn_fs_root_t *fs_root; if (svn_fs_revision_root(&fs_root, fs, revnum, subpool) != SVN_NO_ERROR) return false; svn_boolean_t is_dir; if (svn_fs_is_dir(&is_dir, fs_root, pathname, subpool) != SVN_NO_ERROR) return false; return is_dir; } time_t get_epoch(const char* svn_date) { struct tm tm; memset(&tm, 0, sizeof tm); QByteArray date(svn_date, strlen(svn_date) - 8); strptime(date, "%Y-%m-%dT%H:%M:%S", &tm); return timegm(&tm); } class SvnRevision { public: AprAutoPool pool; QHash transactions; QList allMatchRules; RepositoryHash repositories; IdentityHash identities; QString userdomain; svn_fs_t *fs; svn_fs_root_t *fs_root; int revnum; // must call fetchRevProps first: QByteArray authorident; QByteArray log; uint epoch; bool ruledebug; bool propsFetched; bool needCommit; SvnRevision(int revision, svn_fs_t *f, apr_pool_t *parent_pool) : pool(parent_pool), fs(f), fs_root(0), revnum(revision), propsFetched(false) { ruledebug = CommandLineParser::instance()->contains( QLatin1String("debug-rules")); } int open() { SVN_ERR(svn_fs_revision_root(&fs_root, fs, revnum, pool)); return EXIT_SUCCESS; } int prepareTransactions(); int fetchRevProps(); int commit(); - int exportEntry(const char *path, const svn_fs_path_change_t *change, apr_hash_t *changes); - int exportDispatch(const char *path, const svn_fs_path_change_t *change, + int exportEntry(const char *path, const svn_fs_path_change2_t *change, apr_hash_t *changes); + int exportDispatch(const char *path, const svn_fs_path_change2_t *change, const char *path_from, svn_revnum_t rev_from, apr_hash_t *changes, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules, apr_pool_t *pool); - int exportInternal(const char *path, const svn_fs_path_change_t *change, + int exportInternal(const char *path, const svn_fs_path_change2_t *change, const char *path_from, svn_revnum_t rev_from, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules); - int recurse(const char *path, const svn_fs_path_change_t *change, + int recurse(const char *path, const svn_fs_path_change2_t *change, const char *path_from, const MatchRuleList &matchRules, svn_revnum_t rev_from, apr_hash_t *changes, apr_pool_t *pool); + int addGitIgnore(apr_pool_t *pool, const char *key, QString path, + svn_fs_root_t *fs_root, Repository::Transaction *txn, const char *content = NULL); + int fetchIgnoreProps(QString *ignore, apr_pool_t *pool, const char *key, svn_fs_root_t *fs_root); + int fetchUnknownProps(apr_pool_t *pool, const char *key, svn_fs_root_t *fs_root); +private: + void splitPathName(const Rules::Match &rule, const QString &pathName, QString *svnprefix_p, + QString *repository_p, QString *effectiveRepository_p, QString *branch_p, QString *path_p); }; int SvnPrivate::exportRevision(int revnum) { SvnRevision rev(revnum, fs, global_pool); rev.allMatchRules = allMatchRules; rev.repositories = repositories; rev.identities = identities; rev.userdomain = userdomain; // open this revision: printf("Exporting revision %d ", revnum); fflush(stdout); if (rev.open() == EXIT_FAILURE) return EXIT_FAILURE; if (rev.prepareTransactions() == EXIT_FAILURE) return EXIT_FAILURE; if (!rev.needCommit) { printf(" nothing to do\n"); return EXIT_SUCCESS; // no changes? } if (rev.commit() == EXIT_FAILURE) return EXIT_FAILURE; printf(" done\n"); return EXIT_SUCCESS; } +void SvnRevision::splitPathName(const Rules::Match &rule, const QString &pathName, QString *svnprefix_p, + QString *repository_p, QString *effectiveRepository_p, QString *branch_p, QString *path_p) +{ + QString svnprefix = pathName; + svnprefix.truncate(rule.rx.matchedLength()); + + if (svnprefix_p) { + *svnprefix_p = svnprefix; + } + + if (repository_p) { + *repository_p = svnprefix; + repository_p->replace(rule.rx, rule.repository); + foreach (Rules::Match::Substitution subst, rule.repo_substs) { + subst.apply(*repository_p); + } + } + + if (effectiveRepository_p) { + *effectiveRepository_p = svnprefix; + effectiveRepository_p->replace(rule.rx, rule.repository); + foreach (Rules::Match::Substitution subst, rule.repo_substs) { + subst.apply(*effectiveRepository_p); + } + Repository *repository = repositories.value(*effectiveRepository_p, 0); + if (repository) { + *effectiveRepository_p = repository->getEffectiveRepository()->getName(); + } + } + + if (branch_p) { + *branch_p = svnprefix; + branch_p->replace(rule.rx, rule.branch); + foreach (Rules::Match::Substitution subst, rule.branch_substs) { + subst.apply(*branch_p); + } + } + + if (path_p) { + QString prefix = svnprefix; + prefix.replace(rule.rx, rule.prefix); + *path_p = prefix + pathName.mid(svnprefix.length()); + } +} + int SvnRevision::prepareTransactions() { // find out what was changed in this revision: apr_hash_t *changes; - SVN_ERR(svn_fs_paths_changed(&changes, fs_root, pool)); + SVN_ERR(svn_fs_paths_changed2(&changes, fs_root, pool)); - QMap map; + QMap map; for (apr_hash_index_t *i = apr_hash_first(pool, changes); i; i = apr_hash_next(i)) { const void *vkey; void *value; apr_hash_this(i, &vkey, NULL, &value); const char *key = reinterpret_cast(vkey); - svn_fs_path_change_t *change = reinterpret_cast(value); + svn_fs_path_change2_t *change = reinterpret_cast(value); // If we mix path deletions with path adds/replaces we might erase a // branch after that it has been reset -> history truncated if (map.contains(QByteArray(key))) { // If the same path is deleted and added, we need to put the // deletions into the map first, then the addition. if (change->change_kind == svn_fs_path_change_delete) { // XXX } fprintf(stderr, "\nDuplicate key found in rev %d: %s\n", revnum, key); fprintf(stderr, "This needs more code to be handled, file a bug report\n"); fflush(stderr); exit(1); } map.insertMulti(QByteArray(key), change); } - QMapIterator i(map); + QMapIterator i(map); while (i.hasNext()) { i.next(); if (exportEntry(i.key(), i.value(), changes) == EXIT_FAILURE) return EXIT_FAILURE; } return EXIT_SUCCESS; } int SvnRevision::fetchRevProps() { if( propsFetched ) return EXIT_SUCCESS; apr_hash_t *revprops; SVN_ERR(svn_fs_revision_proplist(&revprops, fs, revnum, pool)); svn_string_t *svnauthor = (svn_string_t*)apr_hash_get(revprops, "svn:author", APR_HASH_KEY_STRING); svn_string_t *svndate = (svn_string_t*)apr_hash_get(revprops, "svn:date", APR_HASH_KEY_STRING); svn_string_t *svnlog = (svn_string_t*)apr_hash_get(revprops, "svn:log", APR_HASH_KEY_STRING); if (svnlog) log = svnlog->data; else log.clear(); authorident = svnauthor ? identities.value(svnauthor->data) : QByteArray(); epoch = svndate ? get_epoch(svndate->data) : 0; if (authorident.isEmpty()) { if (!svnauthor || svn_string_isempty(svnauthor)) authorident = "nobody "; else authorident = svnauthor->data + QByteArray(" <") + svnauthor->data + QByteArray("@") + userdomain.toUtf8() + QByteArray(">"); } propsFetched = true; return EXIT_SUCCESS; } int SvnRevision::commit() { // now create the commit if (fetchRevProps() != EXIT_SUCCESS) return EXIT_FAILURE; foreach (Repository *repo, repositories.values()) { repo->commit(); } foreach (Repository::Transaction *txn, transactions) { txn->setAuthor(authorident); txn->setDateTime(epoch); txn->setLog(log); txn->commit(); delete txn; } return EXIT_SUCCESS; } -int SvnRevision::exportEntry(const char *key, const svn_fs_path_change_t *change, +int SvnRevision::exportEntry(const char *key, const svn_fs_path_change2_t *change, apr_hash_t *changes) { AprAutoPool revpool(pool.data()); QString current = QString::fromUtf8(key); // was this copied from somewhere? svn_revnum_t rev_from = SVN_INVALID_REVNUM; const char *path_from = NULL; if (change->change_kind != svn_fs_path_change_delete) { // svn_fs_copied_from would fail on deleted paths, because the path // obviously no longer exists in the current revision SVN_ERR(svn_fs_copied_from(&rev_from, &path_from, fs_root, key, revpool)); } // is this a directory? svn_boolean_t is_dir; SVN_ERR(svn_fs_is_dir(&is_dir, fs_root, key, revpool)); - if (is_dir) { + // Adding newly created directories + if (is_dir && change->change_kind == svn_fs_path_change_add && path_from == NULL + && CommandLineParser::instance()->contains("empty-dirs")) { + QString keyQString = key; + // Skipping SVN-directory-layout + if (keyQString.endsWith("/trunk") || keyQString.endsWith("/branches") || keyQString.endsWith("/tags")) { + //qDebug() << "Skipping SVN-directory-layout:" << keyQString; + return EXIT_SUCCESS; + } + needCommit = true; + //qDebug() << "Adding directory:" << key; + } + // svn:ignore-properties + else if (is_dir && (change->change_kind == svn_fs_path_change_add || change->change_kind == svn_fs_path_change_modify) + && path_from == NULL && CommandLineParser::instance()->contains("svn-ignore")) { + needCommit = true; + } + else if (is_dir) { if (change->change_kind == svn_fs_path_change_modify || change->change_kind == svn_fs_path_change_add) { if (path_from == NULL) { // freshly added directory, or modified properties // Git doesn't handle directories, so we don't either //qDebug() << " mkdir ignored:" << key; return EXIT_SUCCESS; } qDebug() << " " << key << "was copied from" << path_from << "rev" << rev_from; } else if (change->change_kind == svn_fs_path_change_replace) { if (path_from == NULL) qDebug() << " " << key << "was replaced"; else qDebug() << " " << key << "was replaced from" << path_from << "rev" << rev_from; } else if (change->change_kind == svn_fs_path_change_reset) { qCritical() << " " << key << "was reset, panic!"; return EXIT_FAILURE; } else { // if change_kind == delete, it shouldn't come into this arm of the 'is_dir' test qCritical() << " " << key << "has unhandled change kind " << change->change_kind << ", panic!"; return EXIT_FAILURE; } } else if (change->change_kind == svn_fs_path_change_delete) { is_dir = wasDir(fs, revnum - 1, key, revpool); } if (is_dir) current += '/'; //MultiRule: loop start //Replace all returns with continue, bool isHandled = false; foreach ( const MatchRuleList matchRules, allMatchRules ) { // find the first rule that matches this pathname MatchRuleList::ConstIterator match = findMatchRule(matchRules, revnum, current); if (match != matchRules.constEnd()) { const Rules::Match &rule = *match; if ( exportDispatch(key, change, path_from, rev_from, changes, current, rule, matchRules, revpool) == EXIT_FAILURE ) return EXIT_FAILURE; isHandled = true; } else if (is_dir && path_from != NULL) { qDebug() << current << "is a copy-with-history, auto-recursing"; if ( recurse(key, change, path_from, matchRules, rev_from, changes, revpool) == EXIT_FAILURE ) return EXIT_FAILURE; isHandled = true; } else if (is_dir && change->change_kind == svn_fs_path_change_delete) { qDebug() << current << "deleted, auto-recursing"; if ( recurse(key, change, path_from, matchRules, rev_from, changes, revpool) == EXIT_FAILURE ) return EXIT_FAILURE; isHandled = true; } } if ( isHandled ) { return EXIT_SUCCESS; } if (wasDir(fs, revnum - 1, key, revpool)) { qDebug() << current << "was a directory; ignoring"; } else if (change->change_kind == svn_fs_path_change_delete) { qDebug() << current << "is being deleted but I don't know anything about it; ignoring"; } else { qCritical() << current << "did not match any rules; cannot continue"; return EXIT_FAILURE; } return EXIT_SUCCESS; } -int SvnRevision::exportDispatch(const char *key, const svn_fs_path_change_t *change, +int SvnRevision::exportDispatch(const char *key, const svn_fs_path_change2_t *change, const char *path_from, svn_revnum_t rev_from, apr_hash_t *changes, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules, apr_pool_t *pool) { //if(ruledebug) // qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.lineNumber << "(" << rule.rx.pattern() << ")"; switch (rule.action) { case Rules::Match::Ignore: //if(ruledebug) // qDebug() << " " << "ignoring."; return EXIT_SUCCESS; case Rules::Match::Recurse: if(ruledebug) qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.info() << " " << "recursing."; return recurse(key, change, path_from, matchRules, rev_from, changes, pool); case Rules::Match::Export: if(ruledebug) qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.info() << " " << "exporting."; if (exportInternal(key, change, path_from, rev_from, current, rule, matchRules) == EXIT_SUCCESS) return EXIT_SUCCESS; if (change->change_kind != svn_fs_path_change_delete) { if(ruledebug) qDebug() << "rev" << revnum << qPrintable(current) << "matched rule:" << rule.info() << " " << "Unable to export non path removal."; return EXIT_FAILURE; } // we know that the default action inside recurse is to recurse further or to ignore, // either of which is reasonably safe for deletion qWarning() << "WARN: deleting unknown path" << current << "; auto-recursing"; return recurse(key, change, path_from, matchRules, rev_from, changes, pool); } // never reached return EXIT_FAILURE; } -int SvnRevision::exportInternal(const char *key, const svn_fs_path_change_t *change, +int SvnRevision::exportInternal(const char *key, const svn_fs_path_change2_t *change, const char *path_from, svn_revnum_t rev_from, const QString ¤t, const Rules::Match &rule, const MatchRuleList &matchRules) { needCommit = true; - QString svnprefix, repository, branch, path; - splitPathName(rule, current, &svnprefix, &repository, &branch, &path); + QString svnprefix, repository, effectiveRepository, branch, path; + splitPathName(rule, current, &svnprefix, &repository, &effectiveRepository, &branch, &path); Repository *repo = repositories.value(repository, 0); if (!repo) { if (change->change_kind != svn_fs_path_change_delete) qCritical() << "Rule" << rule << "references unknown repository" << repository; return EXIT_FAILURE; } printf("."); fflush(stdout); // qDebug() << " " << qPrintable(current) << "rev" << revnum << "->" // << qPrintable(repository) << qPrintable(branch) << qPrintable(path); - if (change->change_kind == svn_fs_path_change_delete && current == svnprefix && path.isEmpty()) { + if (change->change_kind == svn_fs_path_change_delete && current == svnprefix && path.isEmpty() && !repo->hasPrefix()) { if(ruledebug) qDebug() << "repository" << repository << "branch" << branch << "deleted"; return repo->deleteBranch(branch, revnum); } QString previous; - QString prevsvnprefix, prevrepository, prevbranch, prevpath; + QString prevsvnprefix, prevrepository, preveffectiverepository, prevbranch, prevpath; if (path_from != NULL) { previous = QString::fromUtf8(path_from); if (wasDir(fs, rev_from, path_from, pool.data())) { previous += '/'; } MatchRuleList::ConstIterator prevmatch = findMatchRule(matchRules, rev_from, previous, NoIgnoreRule); if (prevmatch != matchRules.constEnd()) { splitPathName(*prevmatch, previous, &prevsvnprefix, &prevrepository, - &prevbranch, &prevpath); + &preveffectiverepository, &prevbranch, &prevpath); } else { qWarning() << "WARN: SVN reports a \"copy from\" @" << revnum << "from" << path_from << "@" << rev_from << "but no matching rules found! Ignoring copy, treating as a modification"; path_from = NULL; } } // current == svnprefix => we're dealing with the contents of the whole branch here if (path_from != NULL && current == svnprefix && path.isEmpty()) { if (previous != prevsvnprefix) { // source is not the whole of its branch qDebug() << qPrintable(current) << "is a partial branch of repository" << qPrintable(prevrepository) << "branch" << qPrintable(prevbranch) << "subdir" << qPrintable(prevpath); - } else if (prevrepository != repository) { + } else if (preveffectiverepository != effectiveRepository) { qWarning() << "WARN:" << qPrintable(current) << "rev" << revnum << "is a cross-repository copy (from repository" << qPrintable(prevrepository) << "branch" << qPrintable(prevbranch) << "path" << qPrintable(prevpath) << "rev" << rev_from << ")"; } else if (path != prevpath) { qDebug() << qPrintable(current) << "is a branch copy which renames base directory of all contents" << qPrintable(prevpath) << "to" << qPrintable(path); // FIXME: Handle with fast-import 'file rename' facility // ??? Might need special handling when path == / or prevpath == / } else { if (prevbranch == branch) { // same branch and same repository qDebug() << qPrintable(current) << "rev" << revnum << "is reseating branch" << qPrintable(branch) << "to an earlier revision" << qPrintable(previous) << "rev" << rev_from; } else { // same repository but not same branch // this means this is a plain branch qDebug() << qPrintable(repository) << ": branch" << qPrintable(branch) << "is branching from" << qPrintable(prevbranch); } if (repo->createBranch(branch, revnum, prevbranch, rev_from) == EXIT_FAILURE) return EXIT_FAILURE; if(CommandLineParser::instance()->contains("svn-branches")) { Repository::Transaction *txn = transactions.value(repository + branch, 0); if (!txn) { txn = repo->newTransaction(branch, svnprefix, revnum); if (!txn) return EXIT_FAILURE; transactions.insert(repository + branch, txn); } if(ruledebug) qDebug() << "Create a true SVN copy of branch (" << key << "->" << branch << path << ")"; txn->deleteFile(path); recursiveDumpDir(txn, fs_root, key, path, pool); } if (rule.annotate) { // create an annotated tag fetchRevProps(); repo->createAnnotatedTag(branch, svnprefix, revnum, authorident, epoch, log); } return EXIT_SUCCESS; } } Repository::Transaction *txn = transactions.value(repository + branch, 0); if (!txn) { txn = repo->newTransaction(branch, svnprefix, revnum); if (!txn) return EXIT_FAILURE; transactions.insert(repository + branch, txn); } // // If this path was copied from elsewhere, use it to infer _some_ // merge points. This heuristic is fairly useful for tracking // changes across directory re-organizations and wholesale branch // imports. + // // NOTE(uqs): HACK ALERT! Only merge between head, projects, and user // branches for the FreeBSD repositories. Never merge into stable or // releng, as we only ever cherry-pick changes to those branches. // FIXME: Needs to move into the ruleset ... - if (path_from != NULL && prevrepository == repository && prevbranch != branch && + if (path_from != NULL && preveffectiverepository == effectiveRepository && prevbranch != branch && (branch.startsWith("master") || branch.startsWith("head") || branch.startsWith("projects") || branch.startsWith("user"))) { if(ruledebug) qDebug() << "copy from branch" << prevbranch << "to branch" << branch << "@rev" << rev_from; txn->noteCopyFromBranch (prevbranch, rev_from); } if (change->change_kind == svn_fs_path_change_replace && path_from == NULL) { if(ruledebug) qDebug() << "replaced with empty path (" << branch << path << ")"; txn->deleteFile(path); } if (change->change_kind == svn_fs_path_change_delete) { if(ruledebug) qDebug() << "delete (" << branch << path << ")"; txn->deleteFile(path); } else if (!current.endsWith('/')) { if(ruledebug) qDebug() << "add/change file (" << key << "->" << branch << path << ")"; dumpBlob(txn, fs_root, key, path, pool); } else { if(ruledebug) qDebug() << "add/change dir (" << key << "->" << branch << path << ")"; - txn->deleteFile(path); + + // Check unknown svn-properties + if (((path_from == NULL && change->prop_mod==1) || (path_from != NULL && change->change_kind == svn_fs_path_change_add)) + && CommandLineParser::instance()->contains("propcheck")) { + if (fetchUnknownProps(pool, key, fs_root) != EXIT_SUCCESS) { + qWarning() << "Error checking svn-properties (" << key << ")"; + } + } + + int ignoreSet = false; + + // Add GitIgnore with svn:ignore + if (((path_from == NULL && change->prop_mod==1) || (path_from != NULL && change->change_kind == svn_fs_path_change_add)) + && CommandLineParser::instance()->contains("svn-ignore")) { + QString svnignore; + // TODO: Check if svn:ignore or other property was changed, but always set on copy/rename (path_from != NULL) + if (fetchIgnoreProps(&svnignore, pool, key, fs_root) != EXIT_SUCCESS) { + qWarning() << "Error fetching svn-properties (" << key << ")"; + } else if (!svnignore.isNull()) { + addGitIgnore(pool, key, path, fs_root, txn, svnignore.toStdString().c_str()); + ignoreSet = true; + } + } + + // Add GitIgnore for empty directories (if GitIgnore was not set previously) + if (CommandLineParser::instance()->contains("empty-dirs") && ignoreSet == false) { + if (addGitIgnore(pool, key, path, fs_root, txn) == EXIT_SUCCESS) { + return EXIT_SUCCESS; + } else { + ignoreSet = true; + } + } + + if (ignoreSet == false) { + txn->deleteFile(path); + } recursiveDumpDir(txn, fs_root, key, path, pool); } return EXIT_SUCCESS; } -int SvnRevision::recurse(const char *path, const svn_fs_path_change_t *change, +int SvnRevision::recurse(const char *path, const svn_fs_path_change2_t *change, const char *path_from, const MatchRuleList &matchRules, svn_revnum_t rev_from, apr_hash_t *changes, apr_pool_t *pool) { svn_fs_root_t *fs_root = this->fs_root; if (change->change_kind == svn_fs_path_change_delete) SVN_ERR(svn_fs_revision_root(&fs_root, fs, revnum - 1, pool)); // get the dir listing svn_node_kind_t kind; SVN_ERR(svn_fs_check_path(&kind, fs_root, path, pool)); if(kind == svn_node_none) { qWarning() << "WARN: Trying to recurse using a nonexistant path" << path << ", ignoring"; return EXIT_SUCCESS; } else if(kind != svn_node_dir) { qWarning() << "WARN: Trying to recurse using a non-directory path" << path << ", ignoring"; return EXIT_SUCCESS; } apr_hash_t *entries; SVN_ERR(svn_fs_dir_entries(&entries, fs_root, path, pool)); AprAutoPool dirpool(pool); // While we get a hash, put it in a map for sorted lookup, so we can // repeat the conversions and get the same git commit hashes. QMap map; for (apr_hash_index_t *i = apr_hash_first(pool, entries); i; i = apr_hash_next(i)) { dirpool.clear(); const void *vkey; void *value; apr_hash_this(i, &vkey, NULL, &value); svn_fs_dirent_t *dirent = reinterpret_cast(value); - if (dirent->kind != svn_node_dir) - continue; // not a directory, so can't recurse; skip map.insertMulti(QByteArray(dirent->name), dirent->kind); } QMapIterator i(map); while (i.hasNext()) { dirpool.clear(); i.next(); QByteArray entry = path + QByteArray("/") + i.key(); QByteArray entryFrom; if (path_from) entryFrom = path_from + QByteArray("/") + i.key(); // check if this entry is in the changelist for this revision already - svn_fs_path_change_t *otherchange = - (svn_fs_path_change_t*)apr_hash_get(changes, entry.constData(), APR_HASH_KEY_STRING); + svn_fs_path_change2_t *otherchange = + (svn_fs_path_change2_t*)apr_hash_get(changes, entry.constData(), APR_HASH_KEY_STRING); if (otherchange && otherchange->change_kind == svn_fs_path_change_add) { qDebug() << entry << "rev" << revnum << "is in the change-list, deferring to that one"; continue; } QString current = QString::fromUtf8(entry); if (i.value() == svn_node_dir) current += '/'; // find the first rule that matches this pathname MatchRuleList::ConstIterator match = findMatchRule(matchRules, revnum, current); if (match != matchRules.constEnd()) { if (exportDispatch(entry, change, entryFrom.isNull() ? 0 : entryFrom.constData(), rev_from, changes, current, *match, matchRules, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } else { if (i.value() == svn_node_dir) { qDebug() << current << "rev" << revnum << "did not match any rules; auto-recursing"; if (recurse(entry, change, entryFrom.isNull() ? 0 : entryFrom.constData(), matchRules, rev_from, changes, dirpool) == EXIT_FAILURE) return EXIT_FAILURE; } } } return EXIT_SUCCESS; } + +int SvnRevision::addGitIgnore(apr_pool_t *pool, const char *key, QString path, + svn_fs_root_t *fs_root, Repository::Transaction *txn, const char *content) +{ + // Check for number of subfiles if no content + if (!content) { + apr_hash_t *entries; + SVN_ERR(svn_fs_dir_entries(&entries, fs_root, key, pool)); + // Return if any subfiles + if (apr_hash_count(entries)!=0) { + return EXIT_FAILURE; + } + } + + // Add gitignore-File + QString gitIgnorePath = path + ".gitignore"; + if (content) { + QIODevice *io = txn->addFile(gitIgnorePath, 33188, strlen(content)); + io->write(content); + io->putChar('\n'); + } else { + QIODevice *io = txn->addFile(gitIgnorePath, 33188, 0); + io->putChar('\n'); + } + + return EXIT_SUCCESS; +} + +int SvnRevision::fetchIgnoreProps(QString *ignore, apr_pool_t *pool, const char *key, svn_fs_root_t *fs_root) +{ + // Get svn:ignore + svn_string_t *prop = NULL; + SVN_ERR(svn_fs_node_prop(&prop, fs_root, key, "svn:ignore", pool)); + if (prop) { + *ignore = QString(prop->data); + } else { + *ignore = QString(); + } + + return EXIT_SUCCESS; +} + +int SvnRevision::fetchUnknownProps(apr_pool_t *pool, const char *key, svn_fs_root_t *fs_root) +{ + // Check all properties + apr_hash_t *table; + SVN_ERR(svn_fs_node_proplist(&table, fs_root, key, pool)); + apr_hash_index_t *hi; + void *propVal; + const void *propKey; + for (hi = apr_hash_first(pool, table); hi; hi = apr_hash_next(hi)) { + apr_hash_this(hi, &propKey, NULL, &propVal); + if (strcmp((char*)propKey, "svn:ignore")!=0) { + qWarning() << "WARN: Unknown svn-property" << (char*)propKey << "set to" << ((svn_string_t*)propVal)->data << "for" << key; + } + } + + return EXIT_SUCCESS; +} +