RobbyV2/podman
1
0
Fork 0
mirror of https://github.com/containers/podman.git synced 2025-09-06 22:37:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

podman image scp remote support & podman image scp tagging

Browse Source 
add support for podman-remote image scp as well as direct access via the API. This entailed
a full rework of the layering of image scp functions as well as the usual API plugging and type creation

also, implemented podman image scp tagging. which makes the syntax much more readable and allows users t tag the new image
they are loading to the local/remote machine:

allow users to pass a "new name" for the image they are transferring
`podman tag` as implemented creates a new image im `image list` when tagging, so this does the same
meaning that when transferring images with tags, podman on the remote machine/user will load two images
ex: `podman image scp computer1::alpine computer2::foobar` creates alpine:latest and localhost/foobar on the remote host

implementing tags means removal of the flexible syntax. In the currently released podman image scp, the user can either specify
`podman image scp source::img dest::` or `podman image scp dest:: source::img`. However, with tags this task becomes really hard to check
which is the image (src) and which is the new tag (dst). Removal of that streamlines the arg parsing process

Signed-off-by: Charlie Doern <cdoern@redhat.com>
This commit is contained in:
cdoern
2022-05-23 14:12:48 -04:00
committed by Charlie Doern
parent c66a489b75
commit 6d3520e8b7
25 changed files with 898 additions and 675 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
cmd/podman/images/scp.go
View File

@@ -1,28 +1,12 @@
package images
import (
"context"
"fmt"
"io/ioutil"
urlP "net/url"
"os"
"os/exec"
"os/user"
"strconv"
"strings"
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/cmd/podman/common"
"github.com/containers/podman/v4/cmd/podman/registry"
"github.com/containers/podman/v4/cmd/podman/system/connection"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/utils"
scpD "github.com/dtylman/scp"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
)
var (
@@ -32,7 +16,6 @@ var (
Annotations: map[string]string{
registry.UnshareNSRequired: "",
registry.ParentNSRequired: "",
registry.EngineMode: registry.ABIMode,
},
Long: saveScpDescription,
Short: "securely copy images",
@@ -46,9 +29,6 @@ var (
var (
parentFlags []string
quiet bool
source entities.ImageScpOptions
dest entities.ImageScpOptions
sshInfo entities.ImageScpConnections
)
func init() {
@@ -66,7 +46,6 @@ func scpFlags(cmd *cobra.Command) {
func scp(cmd *cobra.Command, args []string) (finalErr error) {
var (
// TODO add tag support for images
err error
)
for i, val := range os.Args {
@@ -81,288 +60,17 @@ func scp(cmd *cobra.Command, args []string) (finalErr error) {
}
parentFlags = append(parentFlags, val)
}
podman, err := os.Executable()
if err != nil {
return err
}
f, err := ioutil.TempFile("", "podman") // open temp file for load/save output
if err != nil {
return err
}
confR, err := config.NewConfig("") // create a hand made config for the remote engine since we might use remote and native at once
if err != nil {
return errors.Wrapf(err, "could not make config")
src := args[0]
dst := ""
if len(args) > 1 {
dst = args[1]
}
abiEng, err := registry.NewImageEngine(cmd, args) // abi native engine
err = registry.ImageEngine().Scp(registry.Context(), src, dst, parentFlags, quiet)
if err != nil {
return err
}
cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary
if err != nil {
return err
}
locations := []*entities.ImageScpOptions{}
cliConnections := []string{}
var flipConnections bool
for _, arg := range args {
loc, connect, err := parseImageSCPArg(arg)
if err != nil {
return err
}
locations = append(locations, loc)
cliConnections = append(cliConnections, connect...)
}
source = *locations[0]
switch {
case len(locations) > 1:
if flipConnections, err = validateSCPArgs(locations); err != nil {
return err
}
if flipConnections { // the order of cliConnections matters, we need to flip both arrays since the args are parsed separately sometimes.
cliConnections[0], cliConnections[1] = cliConnections[1], cliConnections[0]
locations[0], locations[1] = locations[1], locations[0]
}
dest = *locations[1]
case len(locations) == 1:
switch {
case len(locations[0].Image) == 0:
return errors.Wrapf(define.ErrInvalidArg, "no source image specified")
case len(locations[0].Image) > 0 && !locations[0].Remote && len(locations[0].User) == 0: // if we have podman image scp $IMAGE
return errors.Wrapf(define.ErrInvalidArg, "must specify a destination")
}
}
source.Quiet = quiet
source.File = f.Name() // after parsing the arguments, set the file for the save/load
dest.File = source.File
if err = os.Remove(source.File); err != nil { // remove the file and simply use its name so podman creates the file upon save. avoids umask errors
return err
}
allLocal := true // if we are all localhost, do not validate connections but if we are using one localhost and one non we need to use sshd
for _, val := range cliConnections {
if !strings.Contains(val, "@localhost::") {
allLocal = false
break
}
}
if allLocal {
cliConnections = []string{}
}
var serv map[string]config.Destination
serv, err = GetServiceInformation(cliConnections, cfg)
if err != nil {
return err
}
// TODO: Add podman remote support
confR.Engine = config.EngineConfig{Remote: true, CgroupManager: "cgroupfs", ServiceDestinations: serv} // pass the service dest (either remote or something else) to engine
saveCmd, loadCmd := createCommands(podman)
switch {
case source.Remote: // if we want to load FROM the remote, dest can either be local or remote in this case
err = saveToRemote(source.Image, source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return err
}
if dest.Remote { // we want to load remote -> remote, both source and dest are remote
rep, err := loadToRemote(dest.File, "", sshInfo.URI[1], sshInfo.Identities[1])
if err != nil {
return err
}
fmt.Println(rep)
break
}
err = execPodman(podman, loadCmd)
if err != nil {
return err
}
case dest.Remote: // remote host load, implies source is local
err = execPodman(podman, saveCmd)
if err != nil {
return err
}
rep, err := loadToRemote(source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return err
}
fmt.Println(rep)
if err = os.Remove(source.File); err != nil {
return err
}
// TODO: Add podman remote support
default: // else native load, both source and dest are local and transferring between users
if source.User == "" { // source user has to be set, destination does not
source.User = os.Getenv("USER")
if source.User == "" {
u, err := user.Current()
if err != nil {
return errors.Wrapf(err, "could not obtain user, make sure the environmental variable $USER is set")
}
source.User = u.Username
}
}
err := abiEng.Transfer(context.Background(), source, dest, parentFlags)
if err != nil {
return err
}
}
return nil
}
// loadToRemote takes image and remote connection information. it connects to the specified client
// and copies the saved image dir over to the remote host and then loads it onto the machine
// returns a string containing output or an error
func loadToRemote(localFile string, tag string, url *urlP.URL, iden string) (string, error) {
dial, remoteFile, err := createConnection(url, iden)
if err != nil {
return "", err
}
defer dial.Close()
n, err := scpD.CopyTo(dial, localFile, remoteFile)
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return " ", errors.Wrapf(err, errOut)
}
var run string
if tag != "" {
return "", errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run = podman + " image load --input=" + remoteFile + ";rm " + remoteFile // run ssh image load of the file copied via scp
out, err := connection.ExecRemoteCommand(dial, run)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(out), "\n"), nil
}
// saveToRemote takes image information and remote connection information. it connects to the specified client
// and saves the specified image on the remote machine and then copies it to the specified local location
// returns an error if one occurs.
func saveToRemote(image, localFile string, tag string, uri *urlP.URL, iden string) error {
dial, remoteFile, err := createConnection(uri, iden)
if err != nil {
return err
}
defer dial.Close()
if tag != "" {
return errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run := podman + " image save " + image + " --format=oci-archive --output=" + remoteFile // run ssh image load of the file copied via scp. Files are reverse in this case...
_, err = connection.ExecRemoteCommand(dial, run)
if err != nil {
return err
}
n, err := scpD.CopyFrom(dial, remoteFile, localFile)
if _, conErr := connection.ExecRemoteCommand(dial, "rm "+remoteFile); conErr != nil {
logrus.Errorf("Removing file on endpoint: %v", conErr)
}
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return errors.Wrapf(err, errOut)
}
return nil
}
// makeRemoteFile creates the necessary remote file on the host to
// save or load the image to. returns a string with the file name or an error
func makeRemoteFile(dial *ssh.Client) (string, error) {
run := "mktemp"
remoteFile, err := connection.ExecRemoteCommand(dial, run)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(remoteFile), "\n"), nil
}
// createConnections takes a boolean determining which ssh client to dial
// and returns the dials client, its newly opened remote file, and an error if applicable.
func createConnection(url *urlP.URL, iden string) (*ssh.Client, string, error) {
cfg, err := connection.ValidateAndConfigure(url, iden)
if err != nil {
return nil, "", err
}
dialAdd, err := ssh.Dial("tcp", url.Host, cfg) // dial the client
if err != nil {
return nil, "", errors.Wrapf(err, "failed to connect")
}
file, err := makeRemoteFile(dialAdd)
if err != nil {
return nil, "", err
}
return dialAdd, file, nil
}
// GetSerivceInformation takes the parsed list of hosts to connect to and validates the information
func GetServiceInformation(cliConnections []string, cfg *config.Config) (map[string]config.Destination, error) {
var serv map[string]config.Destination
var url string
var iden string
for i, val := range cliConnections {
splitEnv := strings.SplitN(val, "::", 2)
sshInfo.Connections = append(sshInfo.Connections, splitEnv[0])
if len(splitEnv[1]) != 0 {
err := validateImageName(splitEnv[1])
if err != nil {
return nil, err
}
source.Image = splitEnv[1]
//TODO: actually use the new name given by the user
}
conn, found := cfg.Engine.ServiceDestinations[sshInfo.Connections[i]]
if found {
url = conn.URI
iden = conn.Identity
} else { // no match, warn user and do a manual connection.
url = "ssh://" + sshInfo.Connections[i]
iden = ""
logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location")
}
urlT, err := urlP.Parse(url) // create an actual url to pass to exec command
if err != nil {
return nil, err
}
if urlT.User.Username() == "" {
if urlT.User, err = connection.GetUserInfo(urlT); err != nil {
return nil, err
}
}
sshInfo.URI = append(sshInfo.URI, urlT)
sshInfo.Identities = append(sshInfo.Identities, iden)
}
return serv, nil
}
// execPodman executes the podman save/load command given the podman binary
func execPodman(podman string, command []string) error {
cmd := exec.Command(podman)
utils.CreateSCPCommand(cmd, command[1:])
logrus.Debugf("Executing podman command: %q", cmd)
return cmd.Run()
}
// createCommands forms the podman save and load commands used by SCP
func createCommands(podman string) ([]string, []string) {
var parentString string
quiet := ""
if source.Quiet {
quiet = "-q "
}
if len(parentFlags) > 0 {
parentString = strings.Join(parentFlags, " ") + " " // if there are parent args, an extra space needs to be added
} else {
parentString = strings.Join(parentFlags, " ")
}
loadCmd := strings.Split(fmt.Sprintf("%s %sload %s--input %s", podman, parentString, quiet, dest.File), " ")
saveCmd := strings.Split(fmt.Sprintf("%s %vsave %s--output %s %s", podman, parentString, quiet, source.File, source.Image), " ")
return saveCmd, loadCmd
}

46
cmd/podman/images/scp_test.go
View File

@@ -1,46 +0,0 @@
package images
import (
"testing"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/stretchr/testify/assert"
)
func TestParseSCPArgs(t *testing.T) {
args := []string{"alpine", "root@localhost::"}
var source *entities.ImageScpOptions
var dest *entities.ImageScpOptions
var err error
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.Image, "alpine")
dest, _, err = parseImageSCPArg(args[1])
assert.Nil(t, err)
assert.Equal(t, dest.Image, "")
assert.Equal(t, dest.User, "root")
args = []string{"root@localhost::alpine"}
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.User, "root")
assert.Equal(t, source.Image, "alpine")
args = []string{"charliedoern@192.168.68.126::alpine", "foobar@192.168.68.126::"}
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
dest, _, err = parseImageSCPArg(args[1])
assert.Nil(t, err)
assert.True(t, dest.Remote)
assert.Equal(t, dest.Image, "")
args = []string{"charliedoern@192.168.68.126::alpine"}
source, _, err = parseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
}

88
cmd/podman/images/scp_utils.go
View File

@@ -1,88 +0,0 @@
package images
import (
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/pkg/errors"
)
// parseImageSCPArg returns the valid connection, and source/destination data based off of the information provided by the user
// arg is a string containing one of the cli arguments returned is a filled out source/destination options structs as well as a connections array and an error if applicable
func parseImageSCPArg(arg string) (*entities.ImageScpOptions, []string, error) {
location := entities.ImageScpOptions{}
var err error
cliConnections := []string{}
switch {
case strings.Contains(arg, "@localhost::"): // image transfer between users
location.User = strings.Split(arg, "@")[0]
location, err = validateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
cliConnections = append(cliConnections, arg)
case strings.Contains(arg, "::"):
location, err = validateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
location.Remote = true
cliConnections = append(cliConnections, arg)
default:
location.Image = arg
}
return &location, cliConnections, nil
}
// validateImagePortion is a helper function to validate the image name in an SCP argument
func validateImagePortion(location entities.ImageScpOptions, arg string) (entities.ImageScpOptions, error) {
if remoteArgLength(arg, 1) > 0 {
err := validateImageName(strings.Split(arg, "::")[1])
if err != nil {
return location, err
}
location.Image = strings.Split(arg, "::")[1] // this will get checked/set again once we validate connections
}
return location, nil
}
// validateSCPArgs takes the array of source and destination options and checks for common errors
func validateSCPArgs(locations []*entities.ImageScpOptions) (bool, error) {
if len(locations) > 2 {
return false, errors.Wrapf(define.ErrInvalidArg, "cannot specify more than two arguments")
}
switch {
case len(locations[0].Image) > 0 && len(locations[1].Image) > 0:
return false, errors.Wrapf(define.ErrInvalidArg, "cannot specify an image rename")
case len(locations[0].Image) == 0 && len(locations[1].Image) == 0:
return false, errors.Wrapf(define.ErrInvalidArg, "a source image must be specified")
case len(locations[0].Image) == 0 && len(locations[1].Image) != 0:
if locations[0].Remote && locations[1].Remote {
return true, nil // we need to flip the cliConnections array so the save/load connections are in the right place
}
}
return false, nil
}
// validateImageName makes sure that the image given is valid and no injections are occurring
// we simply use this for error checking, bot setting the image
func validateImageName(input string) error {
// ParseNormalizedNamed transforms a shortname image into its
// full name reference so busybox => docker.io/library/busybox
// we want to keep our shortnames, so only return an error if
// we cannot parse what the user has given us
_, err := reference.ParseNormalizedNamed(input)
return err
}
// remoteArgLength is a helper function to simplify the extracting of host argument data
// returns an int which contains the length of a specified index in a host::image string
func remoteArgLength(input string, side int) int {
if strings.Contains(input, "::") {
return len((strings.Split(input, "::"))[side])
}
return -1
}

111
cmd/podman/system/connection/add.go
View File

@@ -6,21 +6,18 @@ import (
"net"
"net/url"
"os"
"os/user"
"regexp"
"time"
"github.com/containers/common/pkg/completion"
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/cmd/podman/registry"
"github.com/containers/podman/v4/cmd/podman/system"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/terminal"
"github.com/containers/podman/v4/pkg/domain/utils"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
)
var (
@@ -95,7 +92,7 @@ func add(cmd *cobra.Command, args []string) error {
switch uri.Scheme {
case "ssh":
if uri.User.Username() == "" {
if uri.User, err = GetUserInfo(uri); err != nil {
if uri.User, err = utils.GetUserInfo(uri); err != nil {
return err
}
}
@@ -180,32 +177,8 @@ func add(cmd *cobra.Command, args []string) error {
return cfg.Write()
}
func GetUserInfo(uri *url.URL) (*url.Userinfo, error) {
var (
usr *user.User
err error
)
if u, found := os.LookupEnv("_CONTAINERS_ROOTLESS_UID"); found {
usr, err = user.LookupId(u)
if err != nil {
return nil, errors.Wrapf(err, "failed to look up rootless user")
}
} else {
usr, err = user.Current()
if err != nil {
return nil, errors.Wrapf(err, "failed to obtain current user")
}
}
pw, set := uri.User.Password()
if set {
return url.UserPassword(usr.Username, pw), nil
}
return url.User(usr.Username), nil
}
func getUDS(uri *url.URL, iden string) (string, error) {
cfg, err := ValidateAndConfigure(uri, iden)
cfg, err := utils.ValidateAndConfigure(uri, iden)
if err != nil {
return "", errors.Wrapf(err, "failed to validate")
}
@@ -226,7 +199,7 @@ func getUDS(uri *url.URL, iden string) (string, error) {
if v, found := os.LookupEnv("PODMAN_BINARY"); found {
podman = v
}
infoJSON, err := ExecRemoteCommand(dial, podman+" info --format=json")
infoJSON, err := utils.ExecRemoteCommand(dial, podman+" info --format=json")
if err != nil {
return "", err
}
@@ -241,79 +214,3 @@ func getUDS(uri *url.URL, iden string) (string, error) {
}
return info.Host.RemoteSocket.Path, nil
}
// ValidateAndConfigure will take a ssh url and an identity key (rsa and the like) and ensure the information given is valid
// iden iden can be blank to mean no identity key
// once the function validates the information it creates and returns an ssh.ClientConfig.
func ValidateAndConfigure(uri *url.URL, iden string) (*ssh.ClientConfig, error) {
var signers []ssh.Signer
passwd, passwdSet := uri.User.Password()
if iden != "" { // iden might be blank if coming from image scp or if no validation is needed
value := iden
s, err := terminal.PublicKey(value, []byte(passwd))
if err != nil {
return nil, errors.Wrapf(err, "failed to read identity %q", value)
}
signers = append(signers, s)
logrus.Debugf("SSH Ident Key %q %s %s", value, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { // validate ssh information, specifically the unix file socket used by the ssh agent.
logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock)
c, err := net.Dial("unix", sock)
if err != nil {
return nil, err
}
agentSigners, err := agent.NewClient(c).Signers()
if err != nil {
return nil, err
}
signers = append(signers, agentSigners...)
if logrus.IsLevelEnabled(logrus.DebugLevel) {
for _, s := range agentSigners {
logrus.Debugf("SSH Agent Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
}
}
var authMethods []ssh.AuthMethod // now we validate and check for the authorization methods, most notaibly public key authorization
if len(signers) > 0 {
var dedup = make(map[string]ssh.Signer)
for _, s := range signers {
fp := ssh.FingerprintSHA256(s.PublicKey())
if _, found := dedup[fp]; found {
logrus.Debugf("Dedup SSH Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
dedup[fp] = s
}
var uniq []ssh.Signer
for _, s := range dedup {
uniq = append(uniq, s)
}
authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
return uniq, nil
}))
}
if passwdSet { // if password authentication is given and valid, add to the list
authMethods = append(authMethods, ssh.Password(passwd))
}
if len(authMethods) == 0 {
authMethods = append(authMethods, ssh.PasswordCallback(func() (string, error) {
pass, err := terminal.ReadPassword(fmt.Sprintf("%s's login password:", uri.User.Username()))
return string(pass), err
}))
}
tick, err := time.ParseDuration("40s")
if err != nil {
return nil, err
}
cfg := &ssh.ClientConfig{
User: uri.User.Username(),
Auth: authMethods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: tick,
}
return cfg, nil
}

27
cmd/podman/system/connection/shared.go
View File

@@ -1,27 +0,0 @@
package connection
import (
"bytes"
"github.com/pkg/errors"
"golang.org/x/crypto/ssh"
)
// ExecRemoteCommand takes a ssh client connection and a command to run and executes the
// command on the specified client. The function returns the Stdout from the client or the Stderr
func ExecRemoteCommand(dial *ssh.Client, run string) ([]byte, error) {
sess, err := dial.NewSession() // new ssh client session
if err != nil {
return nil, err
}
defer sess.Close()
var buffer bytes.Buffer
var bufferErr bytes.Buffer
sess.Stdout = &buffer // output from client funneled into buffer
sess.Stderr = &bufferErr // err form client funneled into buffer
if err := sess.Run(run); err != nil { // run the command on the ssh client
return nil, errors.Wrapf(err, bufferErr.String())
}
return buffer.Bytes(), nil
}

2
contrib/cirrus/lib.sh
View File

@@ -135,6 +135,7 @@ setup_rootless() {
req_env_vars GOPATH GOSRC SECRET_ENV_RE
ROOTLESS_USER="${ROOTLESS_USER:-some${RANDOM}dude}"
ROOTLESS_UID=""
local rootless_uid
local rootless_gid
@@ -158,6 +159,7 @@ setup_rootless() {
cd $GOSRC || exit 1
# Guarantee independence from specific values
rootless_uid=$[RANDOM+1000]
ROOTLESS_UID=$rootless_uid
rootless_gid=$[RANDOM+1000]
msg "creating $rootless_uid:$rootless_gid $ROOTLESS_USER user"
groupadd -g $rootless_gid $ROOTLESS_USER

4
contrib/cirrus/setup_environment.sh
View File

@@ -186,10 +186,11 @@ esac
# Required to be defined by caller: Are we testing as root or a regular user
case "$PRIV_NAME" in
root)
if [[ "$TEST_FLAVOR" = "sys" ]]; then
if [[ "$TEST_FLAVOR" = "sys" || "$TEST_FLAVOR" = "apiv2" ]]; then
# Used in local image-scp testing
setup_rootless
echo "PODMAN_ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "PODMAN_ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi
;;
rootless)
@@ -203,6 +204,7 @@ esac
if [[ -n "$ROOTLESS_USER" ]]; then
echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi
# Required to be defined by caller: Are we testing podman or podman-remote client

31
pkg/api/handlers/libpod/images.go
View File

@@ -21,7 +21,9 @@ import (
api "github.com/containers/podman/v4/pkg/api/types"
"github.com/containers/podman/v4/pkg/auth"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/domain/entities/reports"
"github.com/containers/podman/v4/pkg/domain/infra/abi"
domainUtils "github.com/containers/podman/v4/pkg/domain/utils"
"github.com/containers/podman/v4/pkg/errorhandling"
"github.com/containers/podman/v4/pkg/util"
utils2 "github.com/containers/podman/v4/utils"
@@ -670,3 +672,32 @@ func ImagesRemove(w http.ResponseWriter, r *http.Request) {
utils.Error(w, http.StatusInternalServerError, errorhandling.JoinErrors(rmErrors))
}
}
func ImageScp(w http.ResponseWriter, r *http.Request) {
decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
query := struct {
Destination string `schema:"destination"`
Quiet bool `schema:"quiet"`
}{
// This is where you can override the golang default value for one of fields
}
if err := decoder.Decode(&query, r.URL.Query()); err != nil {
utils.Error(w, http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String()))
return
}
sourceArg := utils.GetName(r)
rep, source, dest, _, err := domainUtils.ExecuteTransfer(sourceArg, query.Destination, []string{}, query.Quiet)
if err != nil {
utils.Error(w, http.StatusInternalServerError, err)
return
}
if source != nil || dest != nil {
utils.Error(w, http.StatusBadRequest, errors.Wrapf(define.ErrInvalidArg, "cannot use the user transfer function on the remote client"))
return
}
utils.WriteResponse(w, http.StatusOK, &reports.ScpReport{Id: rep.Names[0]})
}

7
pkg/api/handlers/swagger/responses.go
View File

@@ -41,6 +41,13 @@ type imagesLoadResponseLibpod struct {
Body entities.ImageLoadReport
}
// Image Scp
// swagger:response
type imagesScpResponseLibpod struct {
// in:body
Body reports.ScpReport
}
// Image Import
// swagger:response
type imagesImportResponseLibpod struct {

34
pkg/api/server/register_images.go
View File

@@ -1615,5 +1615,39 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
// 500:
// $ref: "#/responses/internalError"
r.Handle(VersionedPath("/libpod/build"), s.APIHandler(compat.BuildImage)).Methods(http.MethodPost)
// swagger:operation POST /libpod/images/scp/{name} libpod ImageScpLibpod
// ---
// tags:
// - images
// summary: Copy an image from one host to another
// description: Copy an image from one host to another
// parameters:
// - in: path
// name: name
// required: true
// description: source connection/image
// type: string
// - in: query
// name: destination
// required: false
// description: dest connection/image
// type: string
// - in: query
// name: quiet
// required: false
// description: quiet output
// type: boolean
// default: false
// produces:
// - application/json
// responses:
// 200:
// $ref: "#/responses/imagesScpResponseLibpod"
// 400:
// $ref: "#/responses/badParamError"
// 500:
// $ref: '#/responses/internalError'
r.Handle(VersionedPath("/libpod/images/scp/{name:.*}"), s.APIHandler(libpod.ImageScp)).Methods(http.MethodPost)
return nil
}

20
pkg/bindings/images/images.go
View File

@@ -346,3 +346,23 @@ func Search(ctx context.Context, term string, options *SearchOptions) ([]entitie
return results, nil
}
func Scp(ctx context.Context, source, destination *string, options ScpOptions) (reports.ScpReport, error) {
rep := reports.ScpReport{}
conn, err := bindings.GetClient(ctx)
if err != nil {
return rep, err
}
params, err := options.ToParams()
if err != nil {
return rep, err
}
response, err := conn.DoRequest(ctx, nil, http.MethodPost, fmt.Sprintf("/images/scp/%s", *source), params, nil)
if err != nil {
return rep, err
}
defer response.Body.Close()
return rep, response.Process(&rep)
}

5
pkg/bindings/images/types.go
View File

@@ -188,3 +188,8 @@ type BuildOptions struct {
// ExistsOptions are optional options for checking if an image exists
type ExistsOptions struct {
}
type ScpOptions struct {
Quiet *bool
Destination *string
}

12
pkg/bindings/images/types_scp_options.go Normal file
View File

@@ -0,0 +1,12 @@
package images
import (
"net/url"
"github.com/containers/podman/v4/pkg/bindings/internal/util"
)
// ToParams formats struct fields to be passed to API service
func (o *ScpOptions) ToParams() (url.Values, error) {
return util.ToParams(o)
}

2
pkg/domain/entities/engine_image.go
View File

@@ -22,12 +22,12 @@ type ImageEngine interface {
Push(ctx context.Context, source string, destination string, opts ImagePushOptions) error
Remove(ctx context.Context, images []string, opts ImageRemoveOptions) (*ImageRemoveReport, []error)
Save(ctx context.Context, nameOrID string, tags []string, options ImageSaveOptions) error
Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error
Search(ctx context.Context, term string, opts ImageSearchOptions) ([]ImageSearchReport, error)
SetTrust(ctx context.Context, args []string, options SetTrustOptions) error
ShowTrust(ctx context.Context, args []string, options ShowTrustOptions) (*ShowTrustReport, error)
Shutdown(ctx context.Context)
Tag(ctx context.Context, nameOrID string, tags []string, options ImageTagOptions) error
Transfer(ctx context.Context, source ImageScpOptions, dest ImageScpOptions, parentFlags []string) error
Tree(ctx context.Context, nameOrID string, options ImageTreeOptions) (*ImageTreeReport, error)
Unmount(ctx context.Context, images []string, options ImageUnmountOptions) ([]*ImageUnmountReport, error)
Untag(ctx context.Context, nameOrID string, tags []string, options ImageUntagOptions) error

2
pkg/domain/entities/images.go
View File

@@ -325,6 +325,8 @@ type ImageScpOptions struct {
Image string `json:"image,omitempty"`
// User is used in conjunction with Transfer to determine if a valid user was given to save from/load into
User string `json:"user,omitempty"`
// Tag is the name to be used for the image on the destination
Tag string `json:"tag,omitempty"`
}
// ImageScpConnections provides the ssh related information used in remote image transfer

5
pkg/domain/entities/reports/scp.go Normal file
View File

@@ -0,0 +1,5 @@
package reports
type ScpReport struct {
Id string `json:"Id"` //nolint:revive,stylecheck
}

153
pkg/domain/infra/abi/images.go
View File

@@ -3,6 +3,7 @@ package abi
import (
"context"
"fmt"
"io/fs"
"io/ioutil"
"net/url"
"os"
@@ -29,7 +30,6 @@ import (
domainUtils "github.com/containers/podman/v4/pkg/domain/utils"
"github.com/containers/podman/v4/pkg/errorhandling"
"github.com/containers/podman/v4/pkg/rootless"
"github.com/containers/podman/v4/utils"
"github.com/containers/storage"
dockerRef "github.com/docker/distribution/reference"
"github.com/opencontainers/go-digest"
@@ -350,22 +350,6 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
}
return pushError
}
// Transfer moves images between root and rootless storage so the user specified in the scp call can access and use the image modified by root
func (ir *ImageEngine) Transfer(ctx context.Context, source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string) error {
if source.User == "" {
return errors.Wrapf(define.ErrInvalidArg, "you must define a user when transferring from root to rootless storage")
}
podman, err := os.Executable()
if err != nil {
return err
}
if rootless.IsRootless() && (len(dest.User) == 0 || dest.User == "root") { // if we are rootless and do not have a destination user we can just use sudo
return transferRootless(source, dest, podman, parentFlags)
}
return transferRootful(source, dest, podman, parentFlags)
}
func (ir *ImageEngine) Tag(ctx context.Context, nameOrID string, tags []string, options entities.ImageTagOptions) error {
// Allow tagging manifest list instead of resolving instances from manifest
lookupOptions := &libimage.LookupImageOptions{ManifestList: true}
@@ -694,53 +678,32 @@ func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entitie
return nil, nil
}
func getSigFilename(sigStoreDirPath string) (string, error) {
sigFileSuffix := 1
sigFiles, err := ioutil.ReadDir(sigStoreDirPath)
if err != nil {
return "", err
}
sigFilenames := make(map[string]bool)
for _, file := range sigFiles {
sigFilenames[file.Name()] = true
}
for {
sigFilename := "signature-" + strconv.Itoa(sigFileSuffix)
if _, exists := sigFilenames[sigFilename]; !exists {
return sigFilename, nil
}
sigFileSuffix++
}
}
func localPathFromURI(url *url.URL) (string, error) {
if url.Scheme != "file" {
return "", errors.Errorf("writing to %s is not supported. Use a supported scheme", url.String())
}
return url.Path, nil
}
// putSignature creates signature and saves it to the signstore file
func putSignature(manifestBlob []byte, mech signature.SigningMechanism, sigStoreDir string, instanceDigest digest.Digest, dockerReference dockerRef.Reference, options entities.SignOptions) error {
newSig, err := signature.SignDockerManifest(manifestBlob, dockerReference.String(), mech, options.SignBy)
func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error {
rep, source, dest, flags, err := domainUtils.ExecuteTransfer(src, dst, parentFlags, quiet)
if err != nil {
return err
}
signatureDir := fmt.Sprintf("%s@%s=%s", sigStoreDir, instanceDigest.Algorithm(), instanceDigest.Hex())
if err := os.MkdirAll(signatureDir, 0751); err != nil {
// The directory is allowed to exist
if !os.IsExist(err) {
if (rep == nil && err == nil) && (source != nil && dest != nil) { // we need to execute the transfer
err := Transfer(ctx, *source, *dest, flags)
if err != nil {
return err
}
}
sigFilename, err := getSigFilename(signatureDir)
return nil
}
func Transfer(ctx context.Context, source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string) error {
if source.User == "" {
return errors.Wrapf(define.ErrInvalidArg, "you must define a user when transferring from root to rootless storage")
}
podman, err := os.Executable()
if err != nil {
return err
}
if err = ioutil.WriteFile(filepath.Join(signatureDir, sigFilename), newSig, 0644); err != nil {
return err
if rootless.IsRootless() && (len(dest.User) == 0 || dest.User == "root") { // if we are rootless and do not have a destination user we can just use sudo
return transferRootless(source, dest, podman, parentFlags)
}
return nil
return transferRootful(source, dest, podman, parentFlags)
}
// TransferRootless creates new podman processes using exec.Command and sudo, transferring images between the given source and destination users
@@ -763,7 +726,7 @@ func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOpt
} else {
cmdSave = exec.Command(podman)
}
cmdSave = utils.CreateSCPCommand(cmdSave, saveCommand)
cmdSave = domainUtils.CreateSCPCommand(cmdSave, saveCommand)
logrus.Debugf("Executing save command: %q", cmdSave)
err := cmdSave.Run()
if err != nil {
@@ -776,8 +739,11 @@ func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOpt
} else {
cmdLoad = exec.Command(podman)
}
cmdLoad = utils.CreateSCPCommand(cmdLoad, loadCommand)
cmdLoad = domainUtils.CreateSCPCommand(cmdLoad, loadCommand)
logrus.Debugf("Executing load command: %q", cmdLoad)
if len(dest.Tag) > 0 {
return domainUtils.ScpTag(cmdLoad, podman, dest)
}
return cmdLoad.Run()
}
@@ -833,11 +799,20 @@ func transferRootful(source entities.ImageScpOptions, dest entities.ImageScpOpti
return err
}
}
err = execPodman(uSave, saveCommand)
_, err = execTransferPodman(uSave, saveCommand, false)
if err != nil {
return err
}
return execPodman(uLoad, loadCommand)
out, err := execTransferPodman(uLoad, loadCommand, (len(dest.Tag) > 0))
if err != nil {
return err
}
if out != nil {
image := domainUtils.ExtractImage(out)
_, err := execTransferPodman(uLoad, []string{podman, "tag", image, dest.Tag}, false)
return err
}
return nil
}
func lookupUser(u string) (*user.User, error) {
@@ -847,10 +822,10 @@ func lookupUser(u string) (*user.User, error) {
return user.Lookup(u)
}
func execPodman(execUser *user.User, command []string) error {
cmdLogin, err := utils.LoginUser(execUser.Username)
func execTransferPodman(execUser *user.User, command []string, needToTag bool) ([]byte, error) {
cmdLogin, err := domainUtils.LoginUser(execUser.Username)
if err != nil {
return err
return nil, err
}
defer func() {
@@ -864,11 +839,11 @@ func execPodman(execUser *user.User, command []string) error {
cmd.Stdout = os.Stdout
uid, err := strconv.ParseInt(execUser.Uid, 10, 32)
if err != nil {
return err
return nil, err
}
gid, err := strconv.ParseInt(execUser.Gid, 10, 32)
if err != nil {
return err
return nil, err
}
cmd.SysProcAttr = &syscall.SysProcAttr{
Credential: &syscall.Credential{
@@ -878,5 +853,55 @@ func execPodman(execUser *user.User, command []string) error {
NoSetGroups: false,
},
}
return cmd.Run()
if needToTag {
cmd.Stdout = nil
return cmd.Output()
}
return nil, cmd.Run()
}
func getSigFilename(sigStoreDirPath string) (string, error) {
sigFileSuffix := 1
sigFiles, err := ioutil.ReadDir(sigStoreDirPath)
if err != nil {
return "", err
}
sigFilenames := make(map[string]bool)
for _, file := range sigFiles {
sigFilenames[file.Name()] = true
}
for {
sigFilename := "signature-" + strconv.Itoa(sigFileSuffix)
if _, exists := sigFilenames[sigFilename]; !exists {
return sigFilename, nil
}
sigFileSuffix++
}
}
func localPathFromURI(url *url.URL) (string, error) {
if url.Scheme != "file" {
return "", errors.Errorf("writing to %s is not supported. Use a supported scheme", url.String())
}
return url.Path, nil
}
// putSignature creates signature and saves it to the signstore file
func putSignature(manifestBlob []byte, mech signature.SigningMechanism, sigStoreDir string, instanceDigest digest.Digest, dockerReference dockerRef.Reference, options entities.SignOptions) error {
newSig, err := signature.SignDockerManifest(manifestBlob, dockerReference.String(), mech, options.SignBy)
if err != nil {
return err
}
signatureDir := fmt.Sprintf("%s@%s=%s", sigStoreDir, instanceDigest.Algorithm(), instanceDigest.Hex())
if err := os.MkdirAll(signatureDir, 0751); err != nil {
// The directory is allowed to exist
if !errors.Is(err, fs.ErrExist) {
return err
}
}
sigFilename, err := getSigFilename(signatureDir)
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(signatureDir, sigFilename), newSig, 0644)
}

26
pkg/domain/infra/tunnel/images.go
View File

@@ -2,6 +2,7 @@ package tunnel
import (
"context"
"fmt"
"io/ioutil"
"os"
"strconv"
@@ -12,7 +13,6 @@ import (
"github.com/containers/common/pkg/config"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/types"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/bindings/images"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/domain/entities/reports"
@@ -123,10 +123,6 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities.
return &entities.ImagePullReport{Images: pulledImages}, nil
}
func (ir *ImageEngine) Transfer(ctx context.Context, source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string) error {
return errors.Wrapf(define.ErrNotImplemented, "cannot use the remote client to transfer images between root and rootless storage")
}
func (ir *ImageEngine) Tag(ctx context.Context, nameOrID string, tags []string, opt entities.ImageTagOptions) error {
options := new(images.TagOptions)
for _, newTag := range tags {
@@ -367,3 +363,23 @@ func (ir *ImageEngine) Shutdown(_ context.Context) {
func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entities.SignOptions) (*entities.SignReport, error) {
return nil, errors.New("not implemented yet")
}
func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error {
options := new(images.ScpOptions)
var destination *string
if len(dst) > 1 {
destination = &dst
}
options.Quiet = &quiet
options.Destination = destination
rep, err := images.Scp(ir.ClientCtx, &src, destination, *options)
if err != nil {
return err
}
fmt.Println("Loaded Image(s):", rep.Id)
return nil
}

581
pkg/domain/utils/scp.go Normal file
View File

@@ -0,0 +1,581 @@
package utils
import (
"bytes"
"fmt"
"io/ioutil"
"net"
"net/url"
"os"
"os/exec"
"os/user"
"strconv"
"strings"
"time"
scpD "github.com/dtylman/scp"
"github.com/containers/common/pkg/config"
"github.com/containers/podman/v4/libpod/define"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/containers/podman/v4/pkg/terminal"
"github.com/docker/distribution/reference"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
)
func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entities.ImageLoadReport, *entities.ImageScpOptions, *entities.ImageScpOptions, []string, error) {
source := entities.ImageScpOptions{}
dest := entities.ImageScpOptions{}
sshInfo := entities.ImageScpConnections{}
report := entities.ImageLoadReport{Names: []string{}}
podman, err := os.Executable()
if err != nil {
return nil, nil, nil, nil, err
}
f, err := ioutil.TempFile("", "podman") // open temp file for load/save output
if err != nil {
return nil, nil, nil, nil, err
}
confR, err := config.NewConfig("") // create a hand made config for the remote engine since we might use remote and native at once
if err != nil {
return nil, nil, nil, nil, errors.Wrapf(err, "could not make config")
}
cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary
if err != nil {
return nil, nil, nil, nil, err
}
locations := []*entities.ImageScpOptions{}
cliConnections := []string{}
args := []string{src}
if len(dst) > 0 {
args = append(args, dst)
}
for _, arg := range args {
loc, connect, err := ParseImageSCPArg(arg)
if err != nil {
return nil, nil, nil, nil, err
}
locations = append(locations, loc)
cliConnections = append(cliConnections, connect...)
}
source = *locations[0]
switch {
case len(locations) > 1:
if err = ValidateSCPArgs(locations); err != nil {
return nil, nil, nil, nil, err
}
dest = *locations[1]
case len(locations) == 1:
switch {
case len(locations[0].Image) == 0:
return nil, nil, nil, nil, errors.Wrapf(define.ErrInvalidArg, "no source image specified")
case len(locations[0].Image) > 0 && !locations[0].Remote && len(locations[0].User) == 0: // if we have podman image scp $IMAGE
return nil, nil, nil, nil, errors.Wrapf(define.ErrInvalidArg, "must specify a destination")
}
}
source.Quiet = quiet
source.File = f.Name() // after parsing the arguments, set the file for the save/load
dest.File = source.File
if err = os.Remove(source.File); err != nil { // remove the file and simply use its name so podman creates the file upon save. avoids umask errors
return nil, nil, nil, nil, err
}
allLocal := true // if we are all localhost, do not validate connections but if we are using one localhost and one non we need to use sshd
for _, val := range cliConnections {
if !strings.Contains(val, "@localhost::") {
allLocal = false
break
}
}
if allLocal {
cliConnections = []string{}
}
var serv map[string]config.Destination
serv, err = GetServiceInformation(&sshInfo, cliConnections, cfg)
if err != nil {
return nil, nil, nil, nil, err
}
confR.Engine = config.EngineConfig{Remote: true, CgroupManager: "cgroupfs", ServiceDestinations: serv} // pass the service dest (either remote or something else) to engine
saveCmd, loadCmd := CreateCommands(source, dest, parentFlags, podman)
switch {
case source.Remote: // if we want to load FROM the remote, dest can either be local or remote in this case
err = SaveToRemote(source.Image, source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return nil, nil, nil, nil, err
}
if dest.Remote { // we want to load remote -> remote, both source and dest are remote
rep, id, err := LoadToRemote(dest, dest.File, "", sshInfo.URI[1], sshInfo.Identities[1])
if err != nil {
return nil, nil, nil, nil, err
}
if len(rep) > 0 {
fmt.Println(rep)
}
if len(id) > 0 {
report.Names = append(report.Names, id)
}
break
}
id, err := ExecPodman(dest, podman, loadCmd)
if err != nil {
return nil, nil, nil, nil, err
}
if len(id) > 0 {
report.Names = append(report.Names, id)
}
case dest.Remote: // remote host load, implies source is local
_, err = ExecPodman(dest, podman, saveCmd)
if err != nil {
return nil, nil, nil, nil, err
}
rep, id, err := LoadToRemote(dest, source.File, "", sshInfo.URI[0], sshInfo.Identities[0])
if err != nil {
return nil, nil, nil, nil, err
}
if len(rep) > 0 {
fmt.Println(rep)
}
if len(id) > 0 {
report.Names = append(report.Names, id)
}
if err = os.Remove(source.File); err != nil {
return nil, nil, nil, nil, err
}
default: // else native load, both source and dest are local and transferring between users
if source.User == "" { // source user has to be set, destination does not
source.User = os.Getenv("USER")
if source.User == "" {
u, err := user.Current()
if err != nil {
return nil, nil, nil, nil, errors.Wrapf(err, "could not obtain user, make sure the environmental variable $USER is set")
}
source.User = u.Username
}
}
return nil, &source, &dest, parentFlags, nil // transfer needs to be done in ABI due to cross issues
}
return &report, nil, nil, nil, nil
}
// CreateSCPCommand takes an existing command, appends the given arguments and returns a configured podman command for image scp
func CreateSCPCommand(cmd *exec.Cmd, command []string) *exec.Cmd {
cmd.Args = append(cmd.Args, command...)
cmd.Env = os.Environ()
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
return cmd
}
// ScpTag is a helper function for native podman to tag an image after a local load from image SCP
func ScpTag(cmd *exec.Cmd, podman string, dest entities.ImageScpOptions) error {
cmd.Stdout = nil
out, err := cmd.Output() // this function captures the output temporarily in order to execute the next command
if err != nil {
return err
}
image := ExtractImage(out)
if cmd.Args[0] == "sudo" { // transferRootless will need the sudo since we are loading to sudo from a user acct
cmd = exec.Command("sudo", podman, "tag", image, dest.Tag)
} else {
cmd = exec.Command(podman, "tag", image, dest.Tag)
}
cmd.Stdout = os.Stdout
return cmd.Run()
}
// ExtractImage pulls out the last line of output from save/load (image id)
func ExtractImage(out []byte) string {
fmt.Println(strings.TrimSuffix(string(out), "\n")) // print output
stringOut := string(out) // get all output
arrOut := strings.Split(stringOut, " ") // split it into an array
return strings.ReplaceAll(arrOut[len(arrOut)-1], "\n", "") // replace the trailing \n
}
// LoginUser starts the user process on the host so that image scp can use systemd-run
func LoginUser(user string) (*exec.Cmd, error) {
sleep, err := exec.LookPath("sleep")
if err != nil {
return nil, err
}
machinectl, err := exec.LookPath("machinectl")
if err != nil {
return nil, err
}
cmd := exec.Command(machinectl, "shell", "-q", user+"@.host", sleep, "inf")
err = cmd.Start()
return cmd, err
}
// loadToRemote takes image and remote connection information. it connects to the specified client
// and copies the saved image dir over to the remote host and then loads it onto the machine
// returns a string containing output or an error
func LoadToRemote(dest entities.ImageScpOptions, localFile string, tag string, url *url.URL, iden string) (string, string, error) {
dial, remoteFile, err := CreateConnection(url, iden)
if err != nil {
return "", "", err
}
defer dial.Close()
n, err := scpD.CopyTo(dial, localFile, remoteFile)
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return " ", "", errors.Wrapf(err, errOut)
}
var run string
if tag != "" {
return "", "", errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run = podman + " image load --input=" + remoteFile + ";rm " + remoteFile // run ssh image load of the file copied via scp
out, err := ExecRemoteCommand(dial, run)
if err != nil {
return "", "", err
}
rep := strings.TrimSuffix(string(out), "\n")
outArr := strings.Split(rep, " ")
id := outArr[len(outArr)-1]
if len(dest.Tag) > 0 { // tag the remote image using the output ID
run = podman + " tag " + id + " " + dest.Tag
_, err = ExecRemoteCommand(dial, run)
if err != nil {
return "", "", err
}
}
return rep, id, nil
}
// saveToRemote takes image information and remote connection information. it connects to the specified client
// and saves the specified image on the remote machine and then copies it to the specified local location
// returns an error if one occurs.
func SaveToRemote(image, localFile string, tag string, uri *url.URL, iden string) error {
dial, remoteFile, err := CreateConnection(uri, iden)
if err != nil {
return err
}
defer dial.Close()
if tag != "" {
return errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
}
podman := os.Args[0]
run := podman + " image save " + image + " --format=oci-archive --output=" + remoteFile // run ssh image load of the file copied via scp. Files are reverse in this case...
_, err = ExecRemoteCommand(dial, run)
if err != nil {
return err
}
n, err := scpD.CopyFrom(dial, remoteFile, localFile)
if _, conErr := ExecRemoteCommand(dial, "rm "+remoteFile); conErr != nil {
logrus.Errorf("Removing file on endpoint: %v", conErr)
}
if err != nil {
errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
return errors.Wrapf(err, errOut)
}
return nil
}
// makeRemoteFile creates the necessary remote file on the host to
// save or load the image to. returns a string with the file name or an error
func MakeRemoteFile(dial *ssh.Client) (string, error) {
run := "mktemp"
remoteFile, err := ExecRemoteCommand(dial, run)
if err != nil {
return "", err
}
return strings.TrimSuffix(string(remoteFile), "\n"), nil
}
// createConnections takes a boolean determining which ssh client to dial
// and returns the dials client, its newly opened remote file, and an error if applicable.
func CreateConnection(url *url.URL, iden string) (*ssh.Client, string, error) {
cfg, err := ValidateAndConfigure(url, iden)
if err != nil {
return nil, "", err
}
dialAdd, err := ssh.Dial("tcp", url.Host, cfg) // dial the client
if err != nil {
return nil, "", errors.Wrapf(err, "failed to connect")
}
file, err := MakeRemoteFile(dialAdd)
if err != nil {
return nil, "", err
}
return dialAdd, file, nil
}
// GetSerivceInformation takes the parsed list of hosts to connect to and validates the information
func GetServiceInformation(sshInfo *entities.ImageScpConnections, cliConnections []string, cfg *config.Config) (map[string]config.Destination, error) {
var serv map[string]config.Destination
var urlS string
var iden string
for i, val := range cliConnections {
splitEnv := strings.SplitN(val, "::", 2)
sshInfo.Connections = append(sshInfo.Connections, splitEnv[0])
conn, found := cfg.Engine.ServiceDestinations[sshInfo.Connections[i]]
if found {
urlS = conn.URI
iden = conn.Identity
} else { // no match, warn user and do a manual connection.
urlS = "ssh://" + sshInfo.Connections[i]
iden = ""
logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location")
}
urlFinal, err := url.Parse(urlS) // create an actual url to pass to exec command
if err != nil {
return nil, err
}
if urlFinal.User.Username() == "" {
if urlFinal.User, err = GetUserInfo(urlFinal); err != nil {
return nil, err
}
}
sshInfo.URI = append(sshInfo.URI, urlFinal)
sshInfo.Identities = append(sshInfo.Identities, iden)
}
return serv, nil
}
// execPodman executes the podman save/load command given the podman binary
func ExecPodman(dest entities.ImageScpOptions, podman string, command []string) (string, error) {
cmd := exec.Command(podman)
CreateSCPCommand(cmd, command[1:])
logrus.Debugf("Executing podman command: %q", cmd)
if strings.Contains(strings.Join(command, " "), "load") { // need to tag
if len(dest.Tag) > 0 {
return "", ScpTag(cmd, podman, dest)
}
cmd.Stdout = nil
out, err := cmd.Output()
if err != nil {
return "", err
}
img := ExtractImage(out)
return img, nil
}
return "", cmd.Run()
}
// createCommands forms the podman save and load commands used by SCP
func CreateCommands(source entities.ImageScpOptions, dest entities.ImageScpOptions, parentFlags []string, podman string) ([]string, []string) {
var parentString string
quiet := ""
if source.Quiet {
quiet = "-q "
}
if len(parentFlags) > 0 {
parentString = strings.Join(parentFlags, " ") + " " // if there are parent args, an extra space needs to be added
} else {
parentString = strings.Join(parentFlags, " ")
}
loadCmd := strings.Split(fmt.Sprintf("%s %sload %s--input %s", podman, parentString, quiet, dest.File), " ")
saveCmd := strings.Split(fmt.Sprintf("%s %vsave %s--output %s %s", podman, parentString, quiet, source.File, source.Image), " ")
return saveCmd, loadCmd
}
// parseImageSCPArg returns the valid connection, and source/destination data based off of the information provided by the user
// arg is a string containing one of the cli arguments returned is a filled out source/destination options structs as well as a connections array and an error if applicable
func ParseImageSCPArg(arg string) (*entities.ImageScpOptions, []string, error) {
location := entities.ImageScpOptions{}
var err error
cliConnections := []string{}
switch {
case strings.Contains(arg, "@localhost::"): // image transfer between users
location.User = strings.Split(arg, "@")[0]
location, err = ValidateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
cliConnections = append(cliConnections, arg)
case strings.Contains(arg, "::"):
location, err = ValidateImagePortion(location, arg)
if err != nil {
return nil, nil, err
}
location.Remote = true
cliConnections = append(cliConnections, arg)
default:
location.Image = arg
}
return &location, cliConnections, nil
}
// validateImagePortion is a helper function to validate the image name in an SCP argument
func ValidateImagePortion(location entities.ImageScpOptions, arg string) (entities.ImageScpOptions, error) {
if RemoteArgLength(arg, 1) > 0 {
err := ValidateImageName(strings.Split(arg, "::")[1])
if err != nil {
return location, err
}
location.Image = strings.Split(arg, "::")[1] // this will get checked/set again once we validate connections
}
return location, nil
}
// validateSCPArgs takes the array of source and destination options and checks for common errors
func ValidateSCPArgs(locations []*entities.ImageScpOptions) error {
if len(locations) > 2 {
return errors.Wrapf(define.ErrInvalidArg, "cannot specify more than two arguments")
}
switch {
case len(locations[0].Image) > 0 && len(locations[1].Image) > 0:
locations[1].Tag = locations[1].Image
locations[1].Image = ""
case len(locations[0].Image) == 0 && len(locations[1].Image) == 0:
return errors.Wrapf(define.ErrInvalidArg, "a source image must be specified")
}
return nil
}
// validateImageName makes sure that the image given is valid and no injections are occurring
// we simply use this for error checking, bot setting the image
func ValidateImageName(input string) error {
// ParseNormalizedNamed transforms a shortname image into its
// full name reference so busybox => docker.io/library/busybox
// we want to keep our shortnames, so only return an error if
// we cannot parse what the user has given us
_, err := reference.ParseNormalizedNamed(input)
return err
}
// remoteArgLength is a helper function to simplify the extracting of host argument data
// returns an int which contains the length of a specified index in a host::image string
func RemoteArgLength(input string, side int) int {
if strings.Contains(input, "::") {
return len((strings.Split(input, "::"))[side])
}
return -1
}
// ExecRemoteCommand takes a ssh client connection and a command to run and executes the
// command on the specified client. The function returns the Stdout from the client or the Stderr
func ExecRemoteCommand(dial *ssh.Client, run string) ([]byte, error) {
sess, err := dial.NewSession() // new ssh client session
if err != nil {
return nil, err
}
defer sess.Close()
var buffer bytes.Buffer
var bufferErr bytes.Buffer
sess.Stdout = &buffer // output from client funneled into buffer
sess.Stderr = &bufferErr // err form client funneled into buffer
if err := sess.Run(run); err != nil { // run the command on the ssh client
return nil, errors.Wrapf(err, bufferErr.String())
}
return buffer.Bytes(), nil
}
func GetUserInfo(uri *url.URL) (*url.Userinfo, error) {
var (
usr *user.User
err error
)
if u, found := os.LookupEnv("_CONTAINERS_ROOTLESS_UID"); found {
usr, err = user.LookupId(u)
if err != nil {
return nil, errors.Wrapf(err, "failed to lookup rootless user")
}
} else {
usr, err = user.Current()
if err != nil {
return nil, errors.Wrapf(err, "failed to obtain current user")
}
}
pw, set := uri.User.Password()
if set {
return url.UserPassword(usr.Username, pw), nil
}
return url.User(usr.Username), nil
}
// ValidateAndConfigure will take a ssh url and an identity key (rsa and the like) and ensure the information given is valid
// iden iden can be blank to mean no identity key
// once the function validates the information it creates and returns an ssh.ClientConfig.
func ValidateAndConfigure(uri *url.URL, iden string) (*ssh.ClientConfig, error) {
var signers []ssh.Signer
passwd, passwdSet := uri.User.Password()
if iden != "" { // iden might be blank if coming from image scp or if no validation is needed
value := iden
s, err := terminal.PublicKey(value, []byte(passwd))
if err != nil {
return nil, errors.Wrapf(err, "failed to read identity %q", value)
}
signers = append(signers, s)
logrus.Debugf("SSH Ident Key %q %s %s", value, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { // validate ssh information, specifically the unix file socket used by the ssh agent.
logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock)
c, err := net.Dial("unix", sock)
if err != nil {
return nil, err
}
agentSigners, err := agent.NewClient(c).Signers()
if err != nil {
return nil, err
}
signers = append(signers, agentSigners...)
if logrus.IsLevelEnabled(logrus.DebugLevel) {
for _, s := range agentSigners {
logrus.Debugf("SSH Agent Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
}
}
var authMethods []ssh.AuthMethod // now we validate and check for the authorization methods, most notaibly public key authorization
if len(signers) > 0 {
var dedup = make(map[string]ssh.Signer)
for _, s := range signers {
fp := ssh.FingerprintSHA256(s.PublicKey())
if _, found := dedup[fp]; found {
logrus.Debugf("Dedup SSH Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
}
dedup[fp] = s
}
var uniq []ssh.Signer
for _, s := range dedup {
uniq = append(uniq, s)
}
authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
return uniq, nil
}))
}
if passwdSet { // if password authentication is given and valid, add to the list
authMethods = append(authMethods, ssh.Password(passwd))
}
if len(authMethods) == 0 {
authMethods = append(authMethods, ssh.PasswordCallback(func() (string, error) {
pass, err := terminal.ReadPassword(fmt.Sprintf("%s's login password:", uri.User.Username()))
return string(pass), err
}))
}
tick, err := time.ParseDuration("40s")
if err != nil {
return nil, err
}
cfg := &ssh.ClientConfig{
User: uri.User.Username(),
Auth: authMethods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: tick,
}
return cfg, nil
}

39
pkg/domain/utils/utils_test.go
View File

@@ -5,6 +5,7 @@ import (
"sort"
"testing"
"github.com/containers/podman/v4/pkg/domain/entities"
"github.com/stretchr/testify/assert"
)
@@ -74,3 +75,41 @@ func TestToURLValues(t *testing.T) {
})
}
}
func TestParseSCPArgs(t *testing.T) {
args := []string{"alpine", "root@localhost::"}
var source *entities.ImageScpOptions
var dest *entities.ImageScpOptions
var err error
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.Image, "alpine")
dest, _, err = ParseImageSCPArg(args[1])
assert.Nil(t, err)
assert.Equal(t, dest.Image, "")
assert.Equal(t, dest.User, "root")
args = []string{"root@localhost::alpine"}
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.Equal(t, source.User, "root")
assert.Equal(t, source.Image, "alpine")
args = []string{"charliedoern@192.168.68.126::alpine", "foobar@192.168.68.126::"}
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
dest, _, err = ParseImageSCPArg(args[1])
assert.Nil(t, err)
assert.True(t, dest.Remote)
assert.Equal(t, dest.Image, "")
args = []string{"charliedoern@192.168.68.126::alpine"}
source, _, err = ParseImageSCPArg(args[0])
assert.Nil(t, err)
assert.True(t, source.Remote)
assert.Equal(t, source.Image, "alpine")
}

13
test/apiv2/12-imagesMore.at
View File

@@ -56,4 +56,17 @@ t GET libpod/images/$IMAGE/json 200 \
t DELETE libpod/images/$IMAGE 200 \
.ExitCode=0
podman pull -q $IMAGE
# test podman image SCP
# ssh needs to work so we can validate that the failure is past argument parsing
podman system connection add --default test ssh://$USER@localhost/run/user/$UID/podman/podman.sock
# should fail but need to check the output...
# status 125 here means that the save/load fails due to
# cirrus weirdness with exec.Command. All of the args have been parsed sucessfully.
t POST "libpod/images/scp/$IMAGE?destination=QA::" 500 \
.cause="exit status 125"
t DELETE libpod/images/$IMAGE 200 \
.ExitCode=0
stop_registry

2
test/apiv2/test-apiv2
View File

@@ -23,6 +23,8 @@ REGISTRY_IMAGE="${PODMAN_TEST_IMAGE_REGISTRY}/${PODMAN_TEST_IMAGE_USER}/registry
###############################################################################
# BEGIN setup
USER=$PODMAN_ROOTLESS_USER
UID=$PODMAN_ROOTLESS_UID
TMPDIR=${TMPDIR:-/tmp}
WORKDIR=$(mktemp --tmpdir -d $ME.tmp.XXXXXX)

11
test/e2e/image_scp_test.go
View File

@@ -50,18 +50,12 @@ var _ = Describe("podman image scp", func() {
})
It("podman image scp bogus image", func() {
if IsRemote() {
Skip("this test is only for non-remote")
}
scp := podmanTest.Podman([]string{"image", "scp", "FOOBAR"})
scp.WaitWithDefaultTimeout()
Expect(scp).Should(ExitWithError())
})
It("podman image scp with proper connection", func() {
if IsRemote() {
Skip("this test is only for non-remote")
}
cmd := []string{"system", "connection", "add",
"--default",
"QA",
@@ -86,7 +80,10 @@ var _ = Describe("podman image scp", func() {
// This tests that the input we are given is validated and prepared correctly
// The error given should either be a missing image (due to testing suite complications) or a no such host timeout on ssh
Expect(scp).Should(ExitWithError())
Expect(scp.ErrorToString()).Should(ContainSubstring("no such host"))
// podman-remote exits with a different error
if !IsRemote() {
Expect(scp.ErrorToString()).Should(ContainSubstring("no such host"))
}
})

24
test/system/120-load.bats
View File

@@ -128,8 +128,24 @@ verify_iid_and_name() {
run_podman image inspect --format '{{.Digest}}' $newname
is "$output" "$src_digest" "Digest of re-fetched image matches original"
# Clean up
# test tagging capability
run_podman untag $IMAGE $newname
run_podman image scp ${notme}@localhost::$newname foobar:123
run_podman image inspect --format '{{.Digest}}' foobar:123
is "$output" "$src_digest" "Digest of re-fetched image matches original"
# remove root img for transfer back with another name
_sudo $PODMAN image rm $newname
# get foobar's ID, for an ID transfer test
run_podman image inspect --format '{{.ID}}' foobar:123
run_podman image scp $output ${notme}@localhost::foobartwo
_sudo $PODMAN image exists foobartwo
# Clean up
_sudo $PODMAN image rm foobartwo
run_podman untag $IMAGE $newname
# Negative test for nonexistent image.
@@ -142,12 +158,6 @@ verify_iid_and_name() {
run_podman 125 image scp $nope ${notme}@localhost::
is "$output" "Error: $nope: image not known.*" "Pushing nonexistent image"
# Negative test for copying to a different name
run_podman 125 image scp $IMAGE ${notme}@localhost::newname:newtag
is "$output" "Error: cannot specify an image rename: invalid argument" \
"Pushing with a different name: not allowed"
# FIXME: any point in copying by image ID? What else should we test?
}

24
utils/utils.go
View File

@@ -243,27 +243,3 @@ func MovePauseProcessToScope(pausePidPath string) {
}
}
}
// CreateSCPCommand takes an existing command, appends the given arguments and returns a configured podman command for image scp
func CreateSCPCommand(cmd *exec.Cmd, command []string) *exec.Cmd {
cmd.Args = append(cmd.Args, command...)
cmd.Env = os.Environ()
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
return cmd
}
// LoginUser starts the user process on the host so that image scp can use systemd-run
func LoginUser(user string) (*exec.Cmd, error) {
sleep, err := exec.LookPath("sleep")
if err != nil {
return nil, err
}
machinectl, err := exec.LookPath("machinectl")
if err != nil {
return nil, err
}
cmd := exec.Command(machinectl, "shell", "-q", user+"@.host", sleep, "inf")
err = cmd.Start()
return cmd, err
}