mischief/openbsd-src
1
0
Fork 0
mirror of https://github.com/openbsd/src.git synced 2025-01-10 06:47:55 -08:00
Code

Stop using rsync --delete when syncing up with the CA repos. Instead

Browse Source 
use the files referenced in the manifests to build up a list of files
to keep and remove anything that is not in the list after doing the
full computation.
OK job@ benno@
This commit is contained in:
claudio 2020-06-24 14:39:21 +00:00
parent 0bc1554c2f
commit 74f3d1527d
2 changed files with 139 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
usr.sbin/rpki-client/extern.h
View File

@ -1,4 +1,4 @@
/* $OpenBSD: extern.h,v 1.29 2020/04/30 13:46:39 deraadt Exp $ */
/* $OpenBSD: extern.h,v 1.30 2020/06/24 14:39:21 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
*
@ -263,6 +263,7 @@ struct stats {
size_t crls; /* revocation lists */
size_t vrps; /* total number of vrps */
size_t uniqs; /* number of unique vrps */
size_t del_files; /* number of files removed in cleanup */
char *talnames;
struct timeval elapsed_time;
struct timeval user_time;

141
usr.sbin/rpki-client/main.c
View File

@ -1,4 +1,4 @@
/* $OpenBSD: main.c,v 1.70 2020/05/18 08:46:39 claudio Exp $ */
/* $OpenBSD: main.c,v 1.71 2020/06/24 14:39:21 claudio Exp $ */
/*
* Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
*
@ -127,6 +127,24 @@ struct entity {
TAILQ_HEAD(entityq, entity);
/*
* Database of all file path accessed during a run.
*/
struct filepath {
RB_ENTRY(filepath) entry;
char *file;
};
static inline int
filepathcmp(struct filepath *a, struct filepath *b)
{
return strcasecmp(a->file, b->file);
}
RB_HEAD(filepath_tree, filepath);
RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
struct filepath_tree fpt = RB_INITIALIZER(&fpt);
/*
* Mark that our subprocesses will never return.
*/
@ -159,6 +177,37 @@ logx(const char *fmt, ...)
}
}
/*
* Functions to lookup which files have been accessed during computation.
*/
static void
filepath_add(char *file)
{
struct filepath *fp;
if ((fp = malloc(sizeof(*fp))) == NULL)
err(1, NULL);
if ((fp->file = strdup(file)) == NULL)
err(1, NULL);
if (RB_INSERT(filepath_tree, &fpt, fp) != NULL) {
/* already in the tree */
free(fp->file);
free(fp);
}
}
static int
filepath_exists(char *file)
{
struct filepath needle;
needle.file = file;
return RB_FIND(filepath_tree, &fpt, &needle) != NULL;
}
RB_GENERATE(filepath_tree, filepath, entry, filepathcmp);
/*
* Resolve the media type of a resource by looking at its suffice.
* Returns the type of RTYPE_EOF if not found.
@ -374,6 +423,7 @@ entityq_add(int fd, struct entityq *q, char *file, enum rtype type,
if ((p->descr = strdup(descr)) == NULL)
err(1, "strdup");
filepath_add(file);
TAILQ_INSERT_TAIL(q, p, entries);
/*
@ -724,7 +774,6 @@ proc_rsync(char *prog, char *bind_addr, int fd, int noop)
i = 0;
args[i++] = (char *)prog;
args[i++] = "-rt";
args[i++] = "--delete";
if (bind_addr != NULL) {
args[i++] = "--address";
args[i++] = (char *)bind_addr;
@ -1365,6 +1414,88 @@ tal_load_default(const char *tals[], size_t max)
return (s);
}
static char **
add_to_del(char **del, size_t *dsz, char *file)
{
size_t i = *dsz;
del = reallocarray(del, i + 1, sizeof(*del));
if (del == NULL)
err(1, "reallocarray");
del[i] = strdup(file);
if (del[i] == NULL)
err(1, "strdup");
*dsz = i + 1;
return del;
}
static size_t
repo_cleanup(const char *cachedir, struct repotab *rt)
{
size_t i, delsz = 0;
char *argv[2], **del = NULL;
FTS *fts;
FTSENT *e;
/* change working directory to the cache directory */
if (chdir(cachedir) == -1)
err(1, "%s: chdir", cachedir);
for (i = 0; i < rt->reposz; i++) {
if (asprintf(&argv[0], "%s/%s", rt->repos[i].host,
rt->repos[i].module) == -1)
err(1, NULL);
argv[1] = NULL;
if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT,
NULL)) == NULL)
err(1, "fts_open");
errno = 0;
while ((e = fts_read(fts)) != NULL) {
switch (e->fts_info) {
case FTS_NSOK:
if (!filepath_exists(e->fts_path))
del = add_to_del(del, &delsz,
e->fts_path);
break;
case FTS_D:
case FTS_DP:
/* TODO empty directory pruning */
break;
case FTS_SL:
case FTS_SLNONE:
warnx("symlink %s", e->fts_path);
del = add_to_del(del, &delsz, e->fts_path);
break;
case FTS_NS:
case FTS_ERR:
warnc(e->fts_errno, "fts_read %s", e->fts_path);
break;
default:
warnx("unhandled[%x] %s", e->fts_info,
e->fts_path);
break;
}
errno = 0;
}
if (errno)
err(1, "fts_read");
if (fts_close(fts) == -1)
err(1, "fts_close");
}
for (i = 0; i < delsz; i++) {
if (unlink(del[i]) == -1)
warn("unlink %s", del[i]);
if (verbose > 1)
logx("deleted %s", del[i]);
free(del[i]);
}
free(del);
return delsz;
}
int
main(int argc, char *argv[])
{
@ -1620,7 +1751,7 @@ main(int argc, char *argv[])
ent = entityq_next(proc, &q);
entity_process(proc, rsync, &stats,
&q, ent, &rt, &eid, &v);
if (verbose > 1)
if (verbose > 2)
fprintf(stderr, "%s\n", ent->uri);
entity_free(ent);
}
@ -1666,6 +1797,8 @@ main(int argc, char *argv[])
if (outputfiles(&v, &stats))
rc = 1;
stats.del_files = repo_cleanup(cachedir, &rt);
logx("Route Origin Authorizations: %zu (%zu failed parse, %zu invalid)",
stats.roas, stats.roas_fail, stats.roas_invalid);
logx("Certificates: %zu (%zu failed parse, %zu invalid)",
@ -1675,10 +1808,10 @@ main(int argc, char *argv[])
stats.mfts, stats.mfts_fail, stats.mfts_stale);
logx("Certificate revocation lists: %zu", stats.crls);
logx("Repositories: %zu", stats.repos);
logx("Files removed: %zu", stats.del_files);
logx("VRP Entries: %zu (%zu unique)", stats.vrps, stats.uniqs);
/* Memory cleanup. */
for (i = 0; i < rt.reposz; i++) {
free(rt.repos[i].host);
free(rt.repos[i].module);