Compare commits
31 Commits
c980dc352a
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
ec8030fc81
|
|||
|
f050dcc94f
|
|||
|
f40a5c9276
|
|||
|
0e80ba0b0d
|
|||
|
a223f04909
|
|||
|
e42ae71862
|
|||
|
58c7f7d1be
|
|||
|
eefb21faff
|
|||
|
941c467fc2
|
|||
|
4dcab777ca
|
|||
|
a169890351
|
|||
|
b70070ba1a
|
|||
|
bd72d740e6
|
|||
|
e43c16adb3
|
|||
|
10200fceb9
|
|||
|
27924013d9
|
|||
|
138bc6d24a
|
|||
|
48179034a7
|
|||
|
f9001ecb9d
|
|||
|
86a6c8acce
|
|||
|
4f6f48247d
|
|||
|
8caba75060
|
|||
|
b35391f1f9
|
|||
|
c5992ef19e
|
|||
|
837cc1bcf4
|
|||
|
87db8a0498
|
|||
|
11513adf48
|
|||
|
aaa4ef61d5
|
|||
|
fec09b6d0b
|
|||
|
2566458e25
|
|||
|
14eb531e4a
|
26
README.md
26
README.md
@@ -1,4 +1,26 @@
|
||||
Unisync is a data synchronisation tool written in python and based on [unison](https://github.com/bcpierce00/unison).
|
||||
The goal is to be able to keep data synchronised between multiple computers without needing to have all the data kept locally while at the same time being able to access everything.
|
||||
I couldn't find a tool to fulfill the requirements I had for a synchronisation tool so I am creating my own as a wrapper around unison.
|
||||
|
||||
The development just started so the documentation will be written later.
|
||||
# Prerequisite
|
||||
|
||||
You need to have the following tools installed.
|
||||
|
||||
Locally :
|
||||
- unison
|
||||
- sshfs
|
||||
- nnn
|
||||
|
||||
Remotely :
|
||||
- unison
|
||||
- cleanlinks and lndir (Should be in `xutils-dev` when using apt)
|
||||
|
||||
# Goal
|
||||
|
||||
Unisync purpose is to keep personal data synchronised between multiple machines without needing to have all the data present an all the machines at the same time. For example you might not need to have your movies on your laptop but still want them on your desktop at home or you might want to keep your old pictures only on a server.
|
||||
Unisync requires you to have a "server" (like a NAS at home) that will store all your data allowing you to only copy what you need when you need it.
|
||||
The issue is that you need to know what data is stored on the server to avoid conflict if creating duplicate files or folders. To address this unisync places a symlink for every file you do not wish to keep locally and allows you to mount the remote filesystem (using sshfs) allowing you to access files that aren't synchronised.
|
||||
|
||||
# Developement
|
||||
|
||||
Unisync was at first a simple bash script but as it grew more complex I started struggling to maintain it which is why I am porting it to python. It will make everything more robust, easier to maintain and to add functionalities.
|
||||
I am in the early stages of the developement process, this should be usable someday (hopefully).
|
||||
|
||||
@@ -10,6 +10,9 @@ requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
unisync = "unisync.main:main"
|
||||
|
||||
[tool.poetry]
|
||||
packages = [{include = "unisync", from = "src"}]
|
||||
|
||||
|
||||
@@ -1,22 +1,36 @@
|
||||
# Copyright (C) 2025 Paul Retourné
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import argparse
|
||||
|
||||
def create_argparser() -> argparse.ArgumentParser:
|
||||
def create_argparser(sync_function, add_function, mount_function) -> argparse.ArgumentParser:
|
||||
"""
|
||||
Creates an argument parser to parse the command line arguments.
|
||||
We use subparsers and set a default function for each to perform the correct action.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='unisync',
|
||||
description='File synchronisation application',
|
||||
epilog="""
|
||||
Copyright © 2025 Paul Retourné.
|
||||
License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>."""
|
||||
epilog="Copyright © 2025-2026 Paul Retourné.\n"
|
||||
"License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
parser.add_argument("local", nargs="?")
|
||||
parser.add_argument("remote", nargs="?")
|
||||
parser.set_defaults(func=sync_function)
|
||||
|
||||
remote_addr_group = parser.add_mutually_exclusive_group()
|
||||
remote_addr_group.add_argument("--ip")
|
||||
remote_addr_group.add_argument("--hostname")
|
||||
|
||||
parser.add_argument("--config", help="Path to the configuration file", metavar="path_to_config")
|
||||
|
||||
subparsers = parser.add_subparsers(help='Actions other than synchronisation')
|
||||
|
||||
parser_add = subparsers.add_parser('add', help='Add files to be synchronised.')
|
||||
parser_add.set_defaults(func=add_function)
|
||||
|
||||
parser_mount = subparsers.add_parser('mount', help='Mount the remote.')
|
||||
parser_mount.set_defaults(func=mount_function)
|
||||
|
||||
return parser
|
||||
|
||||
@@ -1,20 +1,29 @@
|
||||
# Copyright (C) 2025 Paul Retourné
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from configparser import UNNAMED_SECTION
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
import ipaddress
|
||||
import configparser
|
||||
from configparser import UNNAMED_SECTION
|
||||
|
||||
from unisync.defaults import *
|
||||
|
||||
@dataclass
|
||||
class ServerConfig:
|
||||
"""
|
||||
Dataclass keeping the config for connecting to the server
|
||||
"""
|
||||
user: str
|
||||
sshargs: list[str] | None = field(default_factory=list)
|
||||
hostname: str = ""
|
||||
ip: str = ""
|
||||
port: int = 22
|
||||
sshargs: str
|
||||
hostname: str
|
||||
ip: str
|
||||
port: int
|
||||
|
||||
def __post_init__(self):
|
||||
"""
|
||||
Make sure a remote is provided and the ip address is valid
|
||||
"""
|
||||
if self.ip == "" and self.hostname == "":
|
||||
raise ValueError("A remote must be provided (ip or hostname)")
|
||||
|
||||
@@ -26,13 +35,37 @@ class ServerConfig:
|
||||
|
||||
@dataclass
|
||||
class RootsConfig:
|
||||
"""
|
||||
Dataclass keeping the paths to the roots to synchronise
|
||||
"""
|
||||
local: str
|
||||
remote: str
|
||||
|
||||
@dataclass
|
||||
class UnisonConfig:
|
||||
"""
|
||||
Dataclass keeping unison specific configurations
|
||||
"""
|
||||
bools: list
|
||||
values: dict
|
||||
|
||||
@dataclass
|
||||
class OtherConfig:
|
||||
"""
|
||||
Dataclass keeping miscellanous configuration options
|
||||
"""
|
||||
cache_dir_path: Path
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""
|
||||
Main dataclass for the configurations
|
||||
"""
|
||||
server: ServerConfig
|
||||
roots: RootsConfig
|
||||
unison: UnisonConfig
|
||||
other: OtherConfig
|
||||
|
||||
|
||||
def load_config(config_path:str) -> Config:
|
||||
"""
|
||||
@@ -42,22 +75,39 @@ def load_config(config_path:str) -> Config:
|
||||
Returns:
|
||||
Config: A populated Config object containing the loaded config.
|
||||
"""
|
||||
config = configparser.ConfigParser(allow_unnamed_section=True)
|
||||
config = configparser.ConfigParser(allow_unnamed_section=True, allow_no_value=True)
|
||||
config.read(config_path)
|
||||
|
||||
# Check if sections are provided
|
||||
server_section = "Server" if "Server" in config.sections() else UNNAMED_SECTION
|
||||
roots_section = "Roots" if "Roots" in config.sections() else UNNAMED_SECTION
|
||||
other_section = "Other" if "Other" in config.sections() else UNNAMED_SECTION
|
||||
|
||||
server_config = ServerConfig(
|
||||
config.get(server_section, "user"),
|
||||
config.get(server_section, "sshargs", fallback=None),
|
||||
config.get(server_section, "hostname", fallback=None),
|
||||
config.get(server_section, "ip", fallback=None),
|
||||
config.getint(server_section, "port", fallback=None)
|
||||
config.get(server_section, "sshargs", fallback=DEFAULT_SERVER_SSHARGS),
|
||||
config.get(server_section, "hostname", fallback=DEFAULT_SERVER_HOSTNAME),
|
||||
config.get(server_section, "ip", fallback=DEFAULT_SERVER_IP),
|
||||
config.getint(server_section, "port", fallback=DEFAULT_SERVER_PORT)
|
||||
)
|
||||
roots_config = RootsConfig(
|
||||
config.get(roots_section, "local"),
|
||||
config.get(roots_section, "local", fallback=DEFAULT_ROOTS_LOCAL),
|
||||
config.get(roots_section, "remote")
|
||||
)
|
||||
return Config(server_config, roots_config)
|
||||
other_config = OtherConfig(
|
||||
Path(config.get(other_section, "cache_dir_path", fallback=DEFAULT_MISC_CACHE_DIR_PATH)).expanduser()
|
||||
)
|
||||
|
||||
args_bool = list()
|
||||
args_val = dict()
|
||||
if "Unison" in config.sections():
|
||||
for key, val in config.items("Unison"):
|
||||
if key in config["DEFAULT"].keys():
|
||||
continue
|
||||
elif val == "" or val == None:
|
||||
args_bool.append(key)
|
||||
else:
|
||||
args_val[key] = val
|
||||
unison_config = UnisonConfig(args_bool, args_val)
|
||||
|
||||
return Config(server_config, roots_config, unison_config, other_config)
|
||||
|
||||
18
src/unisync/defaults.py
Normal file
18
src/unisync/defaults.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# copyright (c) 2026 paul retourné
|
||||
# spdx-license-identifier: gpl-3.0-or-later
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
# Commented out values are part of the config but are required so there is no defaults.
|
||||
# This allows this file to be a list of all the config options.
|
||||
|
||||
# DEFAULT_SERVER_USER: str = ""
|
||||
DEFAULT_SERVER_SSHARGS: str = ""
|
||||
DEFAULT_SERVER_HOSTNAME: str = ""
|
||||
DEFAULT_SERVER_IP: str = ""
|
||||
DEFAULT_SERVER_PORT: int = 22
|
||||
|
||||
DEFAULT_ROOTS_LOCAL: str = str(Path("~/files").expanduser())
|
||||
# DEFAULT_ROOTS_REMOTE: str = ""
|
||||
|
||||
DEFAULT_MISC_CACHE_DIR_PATH: Path = Path("~/.unisync").expanduser()
|
||||
8
src/unisync/errors.py
Normal file
8
src/unisync/errors.py
Normal file
@@ -0,0 +1,8 @@
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
class RemoteMountedError(BaseException):
|
||||
pass
|
||||
|
||||
class InvalidMountError(BaseException):
|
||||
pass
|
||||
@@ -1,22 +1,46 @@
|
||||
# Copyright (C) 2025 Paul Retourné
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import os
|
||||
from argparser import create_argparser
|
||||
from config import RootsConfig, ServerConfig, Config, load_config
|
||||
from pathlib import Path
|
||||
|
||||
from unisync.argparser import create_argparser
|
||||
from unisync.runners import unisync_sync, unisync_add, unisync_mount
|
||||
from unisync.config import load_config
|
||||
from unisync.synchroniser import Synchroniser
|
||||
from unisync.paths import *
|
||||
|
||||
def main():
|
||||
parser = create_argparser()
|
||||
base_namespace = parser.parse_args()
|
||||
parser = create_argparser(unisync_sync, unisync_add, unisync_mount)
|
||||
cli_args = parser.parse_args()
|
||||
|
||||
config_path = os.path.expanduser("~/.config/unisync/config.ini")
|
||||
if base_namespace.config != None and os.path.isfile(base_namespace.config):
|
||||
config = load_config(base_namespace.config)
|
||||
# Check if --config is set
|
||||
if cli_args.config != None and os.path.isfile(cli_args.config):
|
||||
config = load_config(cli_args.config)
|
||||
elif os.path.isfile(config_path):
|
||||
config = load_config(config_path)
|
||||
else:
|
||||
# TODO make the command line arguments work and override the config options
|
||||
# TODO replace the next line with something to do if no config file is found
|
||||
config = load_config(config_path)
|
||||
pass
|
||||
|
||||
# TODO make the command line arguments work and override the config options
|
||||
|
||||
synchroniser = Synchroniser(
|
||||
config.roots.remote,
|
||||
config.roots.local,
|
||||
config.server.user,
|
||||
config.server.ip if config.server.ip != "" else config.server.hostname,
|
||||
config.server.port,
|
||||
config.unison.bools,
|
||||
config.unison.values
|
||||
)
|
||||
|
||||
paths_manager = PathsManager(Path(config.roots.local), config.other.cache_dir_path)
|
||||
|
||||
cli_args.func(synchroniser, paths_manager)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
113
src/unisync/paths.py
Normal file
113
src/unisync/paths.py
Normal file
@@ -0,0 +1,113 @@
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
|
||||
import os.path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
class PathsManager:
|
||||
|
||||
def __init__(self, local_dir:Path, cache_dir:Path):
|
||||
"""
|
||||
Creates a PathsManager with the necessary data
|
||||
Args:
|
||||
local_dir: Path to the top directory of the synchronisation
|
||||
cache_dir: Path to the cache directory that contains the paths file
|
||||
"""
|
||||
if not local_dir.is_dir():
|
||||
raise ValueError("Invalid local directory")
|
||||
self.local_dir = local_dir
|
||||
|
||||
if not cache_dir.is_dir():
|
||||
raise ValueError("Invalid cache directory")
|
||||
self.cache_dir = cache_dir
|
||||
|
||||
self.paths_file:Path = self.cache_dir / "paths"
|
||||
if not self.paths_file.is_file():
|
||||
raise ValueError("The paths file does not exist")
|
||||
|
||||
|
||||
def user_select_files(self, choice_timeout:int=120) -> list[str]:
|
||||
"""
|
||||
Make the user select files in the top directory.
|
||||
Currently uses nnn for the selection.
|
||||
The goal is to replace it in order to avoid using external programs.
|
||||
Args:
|
||||
choice_timeout: Time given to make choices in nnn
|
||||
Returns:
|
||||
list[str]: The list of paths that was selected relative to the top directory
|
||||
Raise:
|
||||
TimeoutExpired: User took too long to choose
|
||||
CalledProcessError: An unknown error occured during the selection
|
||||
"""
|
||||
command = [
|
||||
"/usr/bin/nnn",
|
||||
"-H",
|
||||
"-p", "-",
|
||||
self.local_dir
|
||||
]
|
||||
nnn_process:subprocess.Popen = subprocess.Popen(command, stdout=subprocess.PIPE)
|
||||
try:
|
||||
ret_code = nnn_process.wait(timeout=choice_timeout)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
print("Choice timeout expired", file=sys.stderr)
|
||||
raise e
|
||||
|
||||
if ret_code != 0:
|
||||
print("File selection failed", file=sys.stderr)
|
||||
raise subprocess.CalledProcessError(1, "File selection failed")
|
||||
|
||||
paths_list:list[str] = []
|
||||
while (next_path := nnn_process.stdout.readline()) != b'':
|
||||
next_path = next_path.decode().strip()
|
||||
# Make the path relative to the top directory
|
||||
next_path = next_path[len(str(self.local_dir)):].lstrip("/")
|
||||
paths_list.append(next_path)
|
||||
return paths_list
|
||||
|
||||
def add_files_to_sync(self):
|
||||
while True:
|
||||
try:
|
||||
paths = self.user_select_files()
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
if input("Timeout expired do you want to retry (y/n): ") != "y":
|
||||
raise
|
||||
self.write_new_paths(paths)
|
||||
|
||||
def get_paths_to_sync(self) -> list[str]:
|
||||
"""
|
||||
Return the paths to synchronise as list.
|
||||
"""
|
||||
paths:list[str] = self.paths_file.read_text().split("\n")
|
||||
if paths[-1] == "":
|
||||
paths.pop()
|
||||
return paths
|
||||
|
||||
def write_new_paths(self, paths:list[str]):
|
||||
"""
|
||||
Writes a list of new paths to the file
|
||||
"""
|
||||
current_paths = self.get_paths_to_sync()
|
||||
paths_to_add = list()
|
||||
# Check if one of the parent is already being synchronised
|
||||
# If so there is no need to add the child path
|
||||
for new_path in paths:
|
||||
is_contained = False
|
||||
for existing in current_paths:
|
||||
common = os.path.commonpath([new_path, existing])
|
||||
if common == existing:
|
||||
is_contained = True
|
||||
break
|
||||
|
||||
if not is_contained and new_path not in paths_to_add:
|
||||
paths_to_add.append(new_path)
|
||||
|
||||
with self.paths_file.open("a") as f:
|
||||
for p in paths_to_add:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
36
src/unisync/runners.py
Normal file
36
src/unisync/runners.py
Normal file
@@ -0,0 +1,36 @@
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
from unisync.synchroniser import Synchroniser
|
||||
from unisync.paths import PathsManager
|
||||
|
||||
|
||||
def unisync_sync(synchroniser:Synchroniser, paths_manager:PathsManager):
|
||||
if synchroniser.create_ssh_master_connection() != 0:
|
||||
print("Connection failed quitting")
|
||||
return 1
|
||||
print("Connected to the remote.")
|
||||
|
||||
synchroniser.sync_files(paths_manager.get_paths_to_sync())
|
||||
synchroniser.sync_links(paths_manager.get_paths_to_sync())
|
||||
|
||||
# TODO check the config options and do or don't do the following
|
||||
synchroniser.update_links()
|
||||
#synchroniser.mount_remote_dir()
|
||||
|
||||
synchroniser.close_ssh_master_connection()
|
||||
|
||||
|
||||
def unisync_add(synchroniser:Synchroniser, paths_manager:PathsManager):
|
||||
if synchroniser.create_ssh_master_connection() != 0:
|
||||
print("Connection failed quitting")
|
||||
return 1
|
||||
print("Connected to the remote.")
|
||||
|
||||
paths_manager.add_files_to_sync()
|
||||
|
||||
synchroniser.close_ssh_master_connection()
|
||||
|
||||
|
||||
def unisync_mount(synchroniser:Synchroniser, paths_manager:PathsManager):
|
||||
synchroniser.mount_remote_dir()
|
||||
@@ -1,4 +1,4 @@
|
||||
# Copyright (C) 2025 Paul Retourné
|
||||
# Copyright (C) 2025-2026 Paul Retourné
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import subprocess
|
||||
@@ -7,6 +7,10 @@ import sys
|
||||
import time
|
||||
import logging
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from unisync.errors import RemoteMountedError, InvalidMountError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class Synchroniser:
|
||||
@@ -131,8 +135,66 @@ class Synchroniser:
|
||||
command.append(remote_root)
|
||||
command.append("-batch")
|
||||
|
||||
print(command)
|
||||
proc = subprocess.Popen(command)
|
||||
ret_code = proc.wait()
|
||||
return ret_code
|
||||
|
||||
def update_links(self, background:bool=True):
|
||||
"""
|
||||
Update the links on the remote.
|
||||
First calls cleanlinks to remove deadlinks and empty directories.
|
||||
Then calls lndir to create the new links.
|
||||
Args:
|
||||
- background: controls if the update is done in the background or waited for
|
||||
"""
|
||||
|
||||
link_update_script = (f"cd {self.remote_dir}/links && "
|
||||
"cleanlinks && "
|
||||
"lndir -withrevinfo -ignorelinks -silent ../.data .;")
|
||||
|
||||
if background:
|
||||
link_background_wrapper = f"nohup bash -c \"{link_update_script}\" > /dev/null 2>&1 < /dev/null &"
|
||||
else:
|
||||
link_background_wrapper = link_update_script
|
||||
|
||||
command = [
|
||||
"/usr/bin/ssh",
|
||||
"-S", self.control_path,
|
||||
f"{self.remote_user}@{self.remote_ip}",
|
||||
"-p", str(self.remote_port),
|
||||
link_background_wrapper
|
||||
]
|
||||
|
||||
link_update_process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
if not background:
|
||||
print("Starting links update.")
|
||||
link_update_process.wait()
|
||||
print("Done")
|
||||
|
||||
def mount_remote_dir(self):
|
||||
"""
|
||||
Mount the remote directory to make the local links work.
|
||||
This is achieved using sshfs.
|
||||
Raise:
|
||||
- RemoteMountedError: The .data directory is already a mount point
|
||||
- InvalidMountError: .data is either not a directory or not empty
|
||||
- subprocess.CalledProcessError: An error occured with sshfs
|
||||
"""
|
||||
# Get the absolute path to the correct .data directory resolving symlinks
|
||||
path_to_mount:Path = Path(f"{self.local}/../.data").resolve()
|
||||
if path_to_mount.is_mount():
|
||||
raise RemoteMountedError
|
||||
# Check if it is an empty directory
|
||||
if not path_to_mount.is_dir() or any(path_to_mount.iterdir()):
|
||||
raise InvalidMountError
|
||||
command = [
|
||||
"/usr/bin/sshfs",
|
||||
"-o", "ControlPath={self.control_path}",
|
||||
"-o", "ServerAliveInterval=15",
|
||||
"-p", str(self.remote_port),
|
||||
f"{self.remote_user}@{self.remote_ip}:{self.remote_dir}/.data",
|
||||
str(path_to_mount)
|
||||
]
|
||||
completed_process = subprocess.run(command)
|
||||
completed_process.check_returncode()
|
||||
|
||||
Reference in New Issue
Block a user